1use super::{
4 align_up_resource, backing_segment_range, bind_lane_stable_slot_projections,
5 compile_program_binding_layouts, compile_submission_wave_domain_layout,
6 compile_submission_wave_reusable_capacity_layouts, contiguous_packing_growth_bytes,
7 free_extent_layout_fingerprint, invalid_resource, lane_stable_layout_key,
8 rollback_free_extent_journal, validate_runtime_descriptor_for_admission,
9 AllocatedDynamicGrowth, AllocationLifetime, AllocationSeal, Arc, AtomicU64, BTreeMap,
10 BackingChunkIdentity, BackingClaimCertificate, BackingPrepareDecision, BackingSegment,
11 BufferRequest, CapacityAvailabilityEpoch, CapacityDomainId, CapacityEntry, CapacityEpochs,
12 CapacityUnits, CapacityVector, DeviceAllocationPermit, DeviceBufferRetention,
13 DeviceCapacityAvailabilitySnapshot, DeviceCapacityBudget, DeviceCapacityReservation,
14 DeviceRuntime, Digest, DynamicBackingBlocker, DynamicBackingClaimOccupancy,
15 DynamicBackingClaimResidency, DynamicBackingClaimScope, DynamicBackingDeferralReason,
16 DynamicBackingDeferred, DynamicBackingPackingEnvelope, DynamicBackingPool,
17 DynamicBackingPoolId, DynamicBackingPoolState, DynamicChunkQuarantineReason,
18 DynamicDeviceCapacityBlocked, DynamicPoolDomainSpec, DynamicPoolGrowthIntent,
19 DynamicPoolGrowthReceipt, DynamicPoolIdleReclaim, DynamicPoolLiveOccupancyStatus,
20 DynamicPoolMaintenanceBoundaryChunk, DynamicPoolMaintenanceBoundaryPool,
21 DynamicPoolMaintenanceBoundaryReceipt, DynamicPoolRebalanceReceipt, DynamicResourceShape,
22 DynamicStorageView, EvaluatedBackingRequest, ExecutionLane, FreeExtentIndex,
23 IdleChunkReclaimCandidate, InvocationLivenessMode, LaneBackingPrepareDecision,
24 LaneStableArenaEntry, LaneStableArenaEvictionCandidate, LaneStableArenaLane,
25 LaneStableArenaSlot, LaneStableArenaSlotLease, LaneStableArenaState,
26 LogicalAdmissionCoordinator, LogicalAdmissionCoordinatorId, LogicalBackingBufferView,
27 LogicalBackingSegmentBinding, LogicalBackingSliceAllocationEvidence,
28 LogicalBackingSliceAuthority, LogicalBackingSliceEvidence, Mutex, Ordering, PendingGrowthGuard,
29 PlanNode, PlannedDynamicGrowth, PreparedBackingClaim, PreparedBackingExtent,
30 PreparedLaneBackingClaim, ProgramBindingLayout, QuarantinedDynamicChunk,
31 RequestStateHazardCoordinator, ResidentChunkBacking, ResidentChunkState, ResourceId,
32 ResourceReservation, ResourceRetentionPolicy, ResourceTransactionIdentity, RunId, Serialize,
33 Sha256, StateInitialization, StaticProvisioningBinding, StepResourceSlotKind,
34 SubmissionWaveDomainCapacityLayout, SubmissionWaveDomainLayout, TransactionId, VNextError,
35 DYNAMIC_POOL_MAINTENANCE_BOUNDARY_SCHEMA_VERSION, NEXT_DYNAMIC_POOL_INSTANCE_ID,
36};
37use crate::vnext::{
38 DeviceCapacityPressure, DynamicPoolResidentPressure, ReusableExecutionBucketId,
39 ReusableExecutionMemoryPlan,
40};
41
42#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
43pub struct DynamicPoolGrowthBatchReceipt {
44 pub(super) coordinator_id: LogicalAdmissionCoordinatorId,
45 pub(super) growths: Vec<DynamicPoolGrowthReceipt>,
46 pub(super) capacity_epoch: u64,
47 #[serde(skip_serializing_if = "Option::is_none")]
48 pub(super) rebalance: Option<DynamicPoolRebalanceReceipt>,
49 #[serde(skip_serializing_if = "Option::is_none")]
50 pub(super) maintenance_boundary: Option<DynamicPoolMaintenanceBoundaryReceipt>,
51}
52
53impl DynamicPoolGrowthBatchReceipt {
54 pub const fn coordinator_id(&self) -> LogicalAdmissionCoordinatorId {
55 self.coordinator_id
56 }
57
58 pub fn growths(&self) -> &[DynamicPoolGrowthReceipt] {
59 &self.growths
60 }
61
62 pub const fn capacity_epoch(&self) -> u64 {
63 self.capacity_epoch
64 }
65
66 pub const fn rebalance(&self) -> Option<&DynamicPoolRebalanceReceipt> {
67 self.rebalance.as_ref()
68 }
69
70 pub const fn maintenance_boundary(&self) -> Option<&DynamicPoolMaintenanceBoundaryReceipt> {
71 self.maintenance_boundary.as_ref()
72 }
73}
74
75pub(in crate::vnext::resource) struct DynamicPoolSet<R>
76where
77 R: DeviceRuntime,
78{
79 pub(in crate::vnext::resource) pools:
80 BTreeMap<DynamicBackingPoolId, Arc<DynamicBackingPool<R>>>,
81 pub(in crate::vnext::resource) domains: Vec<DynamicPoolDomainSpec>,
82 pub(in crate::vnext::resource) nodes: Arc<[PlanNode]>,
83 pub(in crate::vnext::resource) submission_wave_layouts: Vec<Option<SubmissionWaveDomainLayout>>,
84 pub(in crate::vnext::resource) submission_wave_reusable_capacity_layouts:
85 BTreeMap<ReusableExecutionBucketId, Vec<Option<SubmissionWaveDomainCapacityLayout>>>,
86 pub(in crate::vnext::resource) program_binding_layouts:
87 BTreeMap<ReusableExecutionBucketId, Arc<ProgramBindingLayout>>,
88 pub(in crate::vnext::resource) reusable_execution: Option<ReusableExecutionMemoryPlan>,
89 pub(in crate::vnext::resource) request_state_hazards: Arc<RequestStateHazardCoordinator>,
90 pub(in crate::vnext::resource) logical_admission: LogicalAdmissionCoordinator,
91 pub(in crate::vnext::resource) budget: Arc<DeviceCapacityBudget>,
92 lane_stable_arenas: Arc<Mutex<LaneStableArenaState>>,
93 binding: StaticProvisioningBinding,
94 runtime: Arc<R>,
96}
97
98pub(in crate::vnext::resource) struct DynamicPoolRebalanceAttempt {
99 pub(in crate::vnext::resource) boundary: DynamicPoolMaintenanceBoundaryReceipt,
100 pub(in crate::vnext::resource) rebalance: Option<DynamicPoolRebalanceReceipt>,
101}
102
103#[derive(Clone, Copy)]
104enum ReclaimTargetPolicy {
105 PreserveRunnableExtent,
106 PreserveAll,
107}
108
109impl<R> DynamicPoolSet<R>
110where
111 R: DeviceRuntime,
112{
113 pub(in crate::vnext::resource) fn new(
114 runtime: Arc<R>,
115 binding: StaticProvisioningBinding,
116 budget: Arc<DeviceCapacityBudget>,
117 logical_admission: LogicalAdmissionCoordinator,
118 domains: Vec<DynamicPoolDomainSpec>,
119 nodes: Arc<[PlanNode]>,
120 reusable_execution: Option<ReusableExecutionMemoryPlan>,
121 ) -> Result<Self, VNextError> {
122 let request_state_hazards = RequestStateHazardCoordinator::compile(&nodes)?;
123 let submission_wave_layouts = domains
124 .iter()
125 .map(|domain| compile_submission_wave_domain_layout(domain, &nodes))
126 .collect::<Result<Vec<_>, _>>()?;
127 let submission_wave_reusable_capacity_layouts =
128 compile_submission_wave_reusable_capacity_layouts(
129 &domains,
130 &submission_wave_layouts,
131 reusable_execution.as_ref(),
132 )?;
133 let program_binding_layouts = compile_program_binding_layouts(
134 &domains,
135 &nodes,
136 &submission_wave_layouts,
137 &submission_wave_reusable_capacity_layouts,
138 )?
139 .into_iter()
140 .map(|(bucket_id, layout)| (bucket_id, Arc::new(layout)))
141 .collect();
142 let mut pools = BTreeMap::new();
143 for domain in &domains {
144 let instance_id = NEXT_DYNAMIC_POOL_INSTANCE_ID
145 .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
146 current.checked_add(1)
147 })
148 .map_err(|_| invalid_resource("dynamic pool instance id space is exhausted"))?;
149 let pool = Arc::new(DynamicBackingPool {
150 instance_id,
151 domain: domain.clone(),
152 logical_admission: logical_admission.clone(),
153 maintenance: Mutex::new(()),
154 next_extent_generation: AtomicU64::new(1),
155 state: Mutex::new(DynamicBackingPoolState {
156 resident_bytes: 0,
157 pending_growth_bytes: 0,
158 next_chunk_ordinal: 1,
159 next_chunk_generation: 1,
160 chunks: BTreeMap::new(),
161 allocator: FreeExtentIndex::default(),
162 live_occupancy: DynamicPoolLiveOccupancyStatus::default(),
163 quarantined: Vec::new(),
164 poisoned: false,
165 }),
166 });
167 if pools.insert(domain.pool_id().clone(), pool).is_some() {
168 return Err(invalid_resource(
169 "dynamic pool set contains a duplicate pool",
170 ));
171 }
172 }
173 Ok(Self {
174 runtime,
175 binding,
176 budget,
177 logical_admission,
178 domains,
179 pools,
180 nodes,
181 submission_wave_layouts,
182 submission_wave_reusable_capacity_layouts,
183 program_binding_layouts,
184 reusable_execution,
185 request_state_hazards,
186 lane_stable_arenas: Arc::new(Mutex::new(LaneStableArenaState::default())),
187 })
188 }
189
190 pub(in crate::vnext::resource) fn program_binding_layout(
191 &self,
192 bucket_id: &ReusableExecutionBucketId,
193 ) -> Option<&Arc<ProgramBindingLayout>> {
194 self.program_binding_layouts.get(bucket_id)
195 }
196
197 pub(in crate::vnext::resource) const fn maximum_active_sequences(&self) -> u32 {
198 self.binding.maximum_active_sequences()
199 }
200
201 pub(in crate::vnext::resource) fn write_capacity_availability(
202 &self,
203 out: &mut Vec<CapacityAvailabilityEpoch>,
204 ) -> Result<CapacityEpochs, VNextError> {
205 let epochs = self.logical_admission.write_availability_epochs(out)?;
206 self.budget.write_availability_epochs(out)?;
207 debug_assert!(out
208 .windows(2)
209 .all(|pair| pair[0].source() < pair[1].source()));
210 Ok(epochs)
211 }
212
213 pub(in crate::vnext::resource) fn reclaim_idle_chunks_for_pressure(
219 &self,
220 pressure: &DeviceCapacityPressure,
221 capacity_availability: DeviceCapacityAvailabilitySnapshot,
222 excluded_domains: &[CapacityDomainId],
223 protected_immediate: &CapacityVector,
224 protected_packing_envelopes: &[DynamicBackingPackingEnvelope],
225 ) -> Result<DynamicPoolRebalanceAttempt, VNextError> {
226 self.reclaim_idle_chunks(
227 pressure,
228 capacity_availability,
229 excluded_domains,
230 protected_immediate,
231 protected_packing_envelopes,
232 ReclaimTargetPolicy::PreserveRunnableExtent,
233 )
234 }
235
236 fn reclaim_idle_chunks(
237 &self,
238 pressure: &DeviceCapacityPressure,
239 capacity_availability: DeviceCapacityAvailabilitySnapshot,
240 excluded_domains: &[CapacityDomainId],
241 protected_immediate: &CapacityVector,
242 protected_packing_envelopes: &[DynamicBackingPackingEnvelope],
243 target_policy: ReclaimTargetPolicy,
244 ) -> Result<DynamicPoolRebalanceAttempt, VNextError> {
245 if pressure.device_id() != self.runtime.descriptor().id.to_string() {
246 return Err(invalid_resource(
247 "dynamic pool rebalance received pressure for another device",
248 ));
249 }
250 let deficit = pressure
251 .requested_bytes()
252 .checked_sub(pressure.available_bytes())
253 .ok_or_else(|| invalid_resource("dynamic pool pressure has no reclaimable deficit"))?;
254 if deficit == 0 {
255 return Err(invalid_resource(
256 "dynamic pool pressure has an empty reclaimable deficit",
257 ));
258 }
259
260 let excluded_domains = excluded_domains
261 .iter()
262 .copied()
263 .collect::<std::collections::BTreeSet<_>>();
264 let pools = self.pools.values().cloned().collect::<Vec<_>>();
265 let maintenance = pools
266 .iter()
267 .map(|pool| {
268 pool.maintenance
269 .lock()
270 .map_err(|_| invalid_resource("dynamic pool maintenance authority is poisoned"))
271 })
272 .collect::<Result<Vec<_>, _>>()?;
273 let mut states = pools
274 .iter()
275 .map(|pool| {
276 pool.state
277 .lock()
278 .map_err(|_| invalid_resource("dynamic backing pool is poisoned"))
279 })
280 .collect::<Result<Vec<_>, _>>()?;
281 let logical = self.logical_admission.snapshot()?;
282 let used_by_domain = logical
283 .domains()
284 .iter()
285 .map(|domain| (domain.domain(), domain.used().get()))
286 .collect::<BTreeMap<_, _>>();
287 let protected_by_domain = protected_immediate
288 .entries()
289 .iter()
290 .map(|entry| (entry.domain(), entry.units().get()))
291 .collect::<BTreeMap<_, _>>();
292 let protected_packing_by_domain = protected_packing_envelopes
293 .iter()
294 .map(|envelope| (envelope.domain_id(), envelope))
295 .collect::<BTreeMap<_, _>>();
296 if protected_packing_by_domain.len() != protected_packing_envelopes.len() {
297 return Err(invalid_resource(
298 "dynamic backing protection contains duplicate packing domains",
299 ));
300 }
301
302 let mut candidates = Vec::new();
303 let mut boundary_pools = Vec::with_capacity(pools.len());
304 let mut reclaimable_by_pool = vec![0_u64; pools.len()];
305 for (pool_index, (pool, state)) in pools.iter().zip(states.iter()).enumerate() {
306 if state.poisoned {
307 return Err(invalid_resource("dynamic backing pool is fail-closed"));
308 }
309 let growth_target = excluded_domains.contains(&pool.domain.domain_id);
310 let used = used_by_domain
311 .get(&pool.domain.domain_id)
312 .copied()
313 .ok_or_else(|| invalid_resource("dynamic pool domain is absent from admission"))?;
314 let physically_occupied = state
315 .resident_bytes
316 .checked_sub(state.allocator.free_bytes)
317 .ok_or_else(|| {
318 invalid_resource("dynamic pool free bytes exceed resident capacity")
319 })?;
320 if physically_occupied != state.live_occupancy.total().physical_bytes() {
321 return Err(invalid_resource(
322 "dynamic pool allocator occupancy differs from its typed live occupancy",
323 ));
324 }
325 let protected = protected_by_domain
326 .get(&pool.domain.domain_id)
327 .copied()
328 .unwrap_or(0);
329 let mut protected_packing_satisfied = true;
330 let mut protected_chunks = match protected_packing_by_domain.get(&pool.domain.domain_id)
331 {
332 Some(envelope) => {
333 if envelope.pool_id() != pool.domain.pool_id()
334 || envelope.total_bytes()? != protected
335 {
336 return Err(invalid_resource(
337 "dynamic backing byte and packing protection diverged",
338 ));
339 }
340 let mut allocator = state.allocator.clone();
341 let mut chunks = std::collections::BTreeSet::new();
342 for &claim_bytes in envelope.claim_bytes_descending() {
343 let segments = match pool.domain.pool.compatibility().profile().view() {
344 DynamicStorageView::Contiguous => allocator
345 .allocate_contiguous(pool.domain.pool_id(), claim_bytes)?
346 .map(|segment| vec![segment]),
347 DynamicStorageView::PagedRegions { block_bytes } => allocator
348 .allocate_paged(pool.domain.pool_id(), claim_bytes, block_bytes)?,
349 };
350 let Some(segments) = segments else {
351 chunks.clear();
352 break;
353 };
354 chunks.extend(segments.iter().map(BackingSegment::chunk_ordinal));
355 }
356 if chunks.is_empty() && protected != 0 {
357 protected_packing_satisfied = false;
358 }
359 chunks
360 }
361 None => std::collections::BTreeSet::new(),
364 };
365 let may_replace_target =
371 matches!(target_policy, ReclaimTargetPolicy::PreserveRunnableExtent);
372 let replacement_keeper = if growth_target
373 && may_replace_target
374 && !protected_packing_satisfied
375 && used == 0
376 && physically_occupied == 0
377 && state.pending_growth_bytes == 0
378 && matches!(
379 pool.domain.pool.compatibility().profile().view(),
380 DynamicStorageView::Contiguous
381 )
382 && protected_packing_by_domain
383 .get(&pool.domain.domain_id)
384 .is_some_and(|envelope| envelope.claim_bytes_descending() == [protected])
385 && state.chunks.values().all(|chunk| {
386 let bytes = chunk.backing._grant.bytes();
387 chunk.live_segments == 0
388 && Arc::strong_count(&chunk.backing) == 1
389 && chunk.backing.descriptor.size_bytes == bytes
390 && state
391 .allocator
392 .by_offset
393 .get(&(chunk.backing.identity.ordinal(), 0))
394 .is_some_and(|extent| {
395 extent.chunk_generation == chunk.backing.identity.generation()
396 && extent.length_bytes == bytes
397 })
398 }) {
399 state
400 .chunks
401 .values()
402 .max_by_key(|chunk| chunk.backing._grant.bytes())
403 .filter(|chunk| {
404 let bytes = chunk.backing._grant.bytes();
405 bytes >= pool.domain.pool.provisioning().minimum_resident_bytes()
406 && bytes < protected
407 })
408 .map(|chunk| (chunk.backing.identity.clone(), chunk.backing._grant.bytes()))
409 } else {
410 None
411 };
412 if let Some((keeper, _)) = &replacement_keeper {
413 if contiguous_packing_growth_bytes(
418 &state.allocator,
419 pool.domain.pool_id(),
420 &[protected],
421 )? != protected
422 {
423 return Err(invalid_resource(
424 "idle target replacement changed its packing need",
425 ));
426 }
427 protected_chunks.insert(keeper.ordinal());
428 }
429 let excluded_from_reclaim = growth_target && replacement_keeper.is_none();
430 let owned = used.max(physically_occupied);
434 let coherent_runnable_floor = match &replacement_keeper {
435 Some((_, bytes)) => *bytes,
438 None => owned.checked_add(protected).ok_or_else(|| {
439 invalid_resource("dynamic pool protected runnable floor overflows u64")
440 })?,
441 };
442 let resident_floor = pool
443 .domain
444 .pool
445 .provisioning()
446 .minimum_resident_bytes()
447 .max(coherent_runnable_floor);
448 let reclaimable = state.resident_bytes.saturating_sub(resident_floor);
449 reclaimable_by_pool[pool_index] = reclaimable;
450 let mut boundary_chunks = Vec::with_capacity(state.chunks.len());
451 for (&ordinal, chunk) in &state.chunks {
452 let chunk_bytes = chunk.backing._grant.bytes();
453 let full_extent = state.allocator.by_offset.get(&(ordinal, 0));
454 let external_references = Arc::strong_count(&chunk.backing).saturating_sub(1);
455 let protected_packing = protected_chunks.contains(&ordinal);
456 let full_extent_available = chunk.backing.descriptor.size_bytes == chunk_bytes
457 && full_extent.is_some_and(|extent| {
458 extent.chunk_generation == chunk.backing.identity.generation()
459 && extent.length_bytes == chunk_bytes
460 });
461 let resident_floor_allows_reclaim = chunk_bytes <= reclaimable;
462 let reclaim_candidate = !excluded_from_reclaim
463 && state.pending_growth_bytes == 0
464 && (protected_packing_satisfied || replacement_keeper.is_some())
465 && chunk.live_segments == 0
466 && external_references == 0
467 && !protected_packing
468 && resident_floor_allows_reclaim
469 && full_extent_available;
470 boundary_chunks.push(DynamicPoolMaintenanceBoundaryChunk {
471 identity: chunk.backing.identity.clone(),
472 bytes: chunk_bytes,
473 live_segments: chunk.live_segments,
474 external_references,
475 protected_packing,
476 full_extent_available,
477 resident_floor_allows_reclaim,
478 reclaim_candidate,
479 });
480 if reclaim_candidate {
481 candidates.push(IdleChunkReclaimCandidate {
482 pool_index,
483 chunk: chunk.backing.identity.clone(),
484 chunk_bytes,
485 });
486 }
487 }
488 boundary_pools.push(DynamicPoolMaintenanceBoundaryPool {
489 pool_id: pool.domain.pool_id().clone(),
490 domain_id: pool.domain.domain_id,
491 excluded_from_reclaim,
492 resident_bytes: state.resident_bytes,
493 pending_growth_bytes: state.pending_growth_bytes,
494 free_bytes: state.allocator.free_bytes,
495 largest_contiguous_bytes: state.allocator.largest_contiguous_bytes(),
496 free_extent_layout_fingerprint: free_extent_layout_fingerprint(&state.allocator),
497 logical_used_bytes: used,
498 live_occupancy: state.live_occupancy,
499 minimum_resident_bytes: pool.domain.pool.provisioning().minimum_resident_bytes(),
500 maximum_resident_bytes: pool.domain.pool.provisioning().maximum_resident_bytes(),
501 protected_immediate_bytes: protected,
502 protected_packing_satisfied,
503 idle_target_replacement_kept_chunk: replacement_keeper.map(|(keeper, _)| keeper),
504 coherent_runnable_floor_bytes: coherent_runnable_floor,
505 resident_floor_bytes: resident_floor,
506 reclaimable_bytes: reclaimable,
507 chunks: boundary_chunks,
508 });
509 }
510
511 let reclaim_candidate_chunks = candidates.len();
512 let reclaim_candidate_bytes = candidates.iter().try_fold(0_u64, |total, candidate| {
513 total
514 .checked_add(candidate.chunk_bytes)
515 .ok_or_else(|| invalid_resource("dynamic reclaim candidate bytes overflow u64"))
516 })?;
517 let mut selected = Vec::<IdleChunkReclaimCandidate>::new();
518 let mut selected_by_pool = vec![0_u64; pools.len()];
519 let mut reclaimed_bytes = 0_u64;
520 let best_single = candidates
521 .iter()
522 .filter(|candidate| candidate.chunk_bytes >= deficit)
523 .min_by(|left, right| {
524 left.chunk_bytes
525 .cmp(&right.chunk_bytes)
526 .then_with(|| left.pool_index.cmp(&right.pool_index))
527 .then_with(|| right.chunk.ordinal().cmp(&left.chunk.ordinal()))
528 })
529 .cloned();
530 if let Some(candidate) = best_single {
531 selected_by_pool[candidate.pool_index] = candidate.chunk_bytes;
532 reclaimed_bytes = candidate.chunk_bytes;
533 selected.push(candidate);
534 } else {
535 candidates.sort_by(|left, right| {
536 right
537 .chunk_bytes
538 .cmp(&left.chunk_bytes)
539 .then_with(|| left.pool_index.cmp(&right.pool_index))
540 .then_with(|| right.chunk.ordinal().cmp(&left.chunk.ordinal()))
541 });
542 for candidate in candidates {
543 let next_pool_total = selected_by_pool[candidate.pool_index]
544 .checked_add(candidate.chunk_bytes)
545 .ok_or_else(|| invalid_resource("dynamic reclaim bytes overflow u64"))?;
546 if next_pool_total > reclaimable_by_pool[candidate.pool_index] {
547 continue;
548 }
549 selected_by_pool[candidate.pool_index] = next_pool_total;
550 reclaimed_bytes = reclaimed_bytes
551 .checked_add(candidate.chunk_bytes)
552 .ok_or_else(|| invalid_resource("dynamic reclaim bytes overflow u64"))?;
553 selected.push(candidate);
554 if reclaimed_bytes >= deficit {
555 break;
556 }
557 }
558 }
559 selected.sort_by(|left, right| {
560 left.pool_index
561 .cmp(&right.pool_index)
562 .then_with(|| left.chunk.ordinal().cmp(&right.chunk.ordinal()))
563 });
564 let mut planned_domains = excluded_domains.iter().copied().collect::<Vec<_>>();
565 planned_domains.sort_unstable();
566 let mut protected_packing_envelopes = protected_packing_envelopes.to_vec();
567 protected_packing_envelopes.sort_by_key(DynamicBackingPackingEnvelope::domain_id);
568 let boundary = DynamicPoolMaintenanceBoundaryReceipt {
569 schema_version: DYNAMIC_POOL_MAINTENANCE_BOUNDARY_SCHEMA_VERSION,
570 coordinator_id: logical.coordinator_id(),
571 logical_release_epoch: logical.release_epoch(),
572 logical_capacity_epoch: logical.capacity_epoch(),
573 plan_device_capacity_epoch: capacity_availability.plan_epoch(),
574 process_device_capacity_epoch: capacity_availability.process_epoch(),
575 pressure: pressure.clone(),
576 planned_domains,
577 protected_immediate: protected_immediate.clone(),
578 protected_packing_envelopes,
579 pools: boundary_pools,
580 reclaim_candidate_chunks,
581 reclaim_candidate_bytes,
582 selected_chunks: selected
583 .iter()
584 .map(|candidate| candidate.chunk.clone())
585 .collect(),
586 selected_bytes: reclaimed_bytes,
587 reclaim_sufficient: reclaimed_bytes >= deficit,
588 };
589 if reclaimed_bytes < deficit {
590 return Ok(DynamicPoolRebalanceAttempt {
591 boundary,
592 rebalance: None,
593 });
594 }
595
596 for candidate in &selected {
597 let state = &states[candidate.pool_index];
598 let chunk = state
599 .chunks
600 .get(&candidate.chunk.ordinal())
601 .ok_or_else(|| invalid_resource("selected dynamic reclaim chunk disappeared"))?;
602 let extent = state
603 .allocator
604 .by_offset
605 .get(&(candidate.chunk.ordinal(), 0))
606 .ok_or_else(|| invalid_resource("selected dynamic reclaim extent disappeared"))?;
607 if chunk.live_segments != 0
608 || Arc::strong_count(&chunk.backing) != 1
609 || chunk.backing.identity != candidate.chunk
610 || extent.chunk_generation != candidate.chunk.generation()
611 || extent.length_bytes != candidate.chunk_bytes
612 {
613 return Err(invalid_resource(
614 "selected dynamic reclaim chunk changed before publication",
615 ));
616 }
617 }
618
619 let published_totals = states
620 .iter()
621 .zip(&selected_by_pool)
622 .map(|(state, &selected_bytes)| {
623 state
624 .resident_bytes
625 .checked_sub(selected_bytes)
626 .ok_or_else(|| invalid_resource("dynamic reclaim resident bytes underflow"))
627 })
628 .collect::<Result<Vec<_>, _>>()?;
629 let mut removed = Vec::with_capacity(selected.len());
630 for candidate in &selected {
631 let state = &mut states[candidate.pool_index];
632 let extent = state
633 .allocator
634 .remove_extent(candidate.chunk.ordinal(), 0)
635 .expect("validated idle chunk retains its exact full extent");
636 debug_assert_eq!(extent.chunk_generation, candidate.chunk.generation());
637 debug_assert_eq!(extent.length_bytes, candidate.chunk_bytes);
638 let chunk = state
639 .chunks
640 .remove(&candidate.chunk.ordinal())
641 .expect("validated idle chunk remains resident");
642 removed.push((candidate.clone(), chunk));
643 }
644 let updates = selected_by_pool
645 .iter()
646 .enumerate()
647 .filter(|(_, selected_bytes)| **selected_bytes != 0)
648 .map(|(pool_index, _)| {
649 (
650 pools[pool_index].domain.domain_id,
651 CapacityUnits::new(published_totals[pool_index]),
652 )
653 })
654 .collect::<Vec<_>>();
655 let epochs = match self.logical_admission.set_domain_totals(&updates) {
656 Ok(epochs) => epochs,
657 Err(error) => {
658 for (candidate, chunk) in removed.drain(..).rev() {
659 let state = &mut states[candidate.pool_index];
660 state
661 .allocator
662 .insert_extent(
663 candidate.chunk.ordinal(),
664 candidate.chunk.generation(),
665 0,
666 candidate.chunk_bytes,
667 )
668 .expect("unpublished idle chunk extent can be restored");
669 assert!(state
670 .chunks
671 .insert(candidate.chunk.ordinal(), chunk)
672 .is_none());
673 }
674 return Err(error);
675 }
676 };
677 for (state, &published_total) in states.iter_mut().zip(&published_totals) {
678 state.resident_bytes = published_total;
679 }
680
681 let mut pool_receipts = Vec::new();
682 for (pool_index, &pool_reclaimed_bytes) in selected_by_pool.iter().enumerate() {
683 if pool_reclaimed_bytes == 0 {
684 continue;
685 }
686 pool_receipts.push(DynamicPoolIdleReclaim {
687 pool_id: pools[pool_index].domain.pool_id().clone(),
688 chunks: selected
689 .iter()
690 .filter(|candidate| candidate.pool_index == pool_index)
691 .map(|candidate| candidate.chunk.clone())
692 .collect(),
693 reclaimed_bytes: pool_reclaimed_bytes,
694 published_capacity_bytes: published_totals[pool_index],
695 });
696 }
697 let reclaimed_chunks = selected.len();
698 drop(states);
699 drop(maintenance);
700 drop(removed);
701 let availability = self.budget.availability_snapshot()?;
702 Ok(DynamicPoolRebalanceAttempt {
703 boundary,
704 rebalance: Some(DynamicPoolRebalanceReceipt {
705 pools: pool_receipts,
706 reclaimed_chunks,
707 reclaimed_bytes,
708 logical_capacity_epoch: epochs.capacity_epoch(),
709 plan_device_capacity_epoch: availability.plan_epoch(),
710 process_device_capacity_epoch: availability.process_epoch(),
711 }),
712 })
713 }
714
715 pub(in crate::vnext::resource) fn maintain_pools(
716 &self,
717 intents: Vec<DynamicPoolGrowthIntent>,
718 ) -> Result<DynamicPoolGrowthBatchReceipt, VNextError> {
719 let mut ignored_capacity_block = None;
720 self.maintain_pools_observed(intents, &mut ignored_capacity_block)
721 }
722
723 pub(in crate::vnext::resource) fn maintain_pools_observed(
724 &self,
725 mut intents: Vec<DynamicPoolGrowthIntent>,
726 capacity_blocked: &mut Option<DynamicDeviceCapacityBlocked>,
727 ) -> Result<DynamicPoolGrowthBatchReceipt, VNextError> {
728 intents.sort_by(|left, right| left.pool_id().cmp(right.pool_id()));
729 if intents
730 .windows(2)
731 .any(|pair| pair[0].pool_id() == pair[1].pool_id())
732 {
733 return Err(invalid_resource(
734 "dynamic maintenance batch contains a duplicate pool",
735 ));
736 }
737 let pools = intents
738 .iter()
739 .map(|intent| {
740 self.pools.get(intent.pool_id()).cloned().ok_or_else(|| {
741 invalid_resource("dynamic maintenance references an unknown pool")
742 })
743 })
744 .collect::<Result<Vec<_>, _>>()?;
745 let _maintenance = pools
746 .iter()
747 .map(|pool| {
748 pool.maintenance
749 .lock()
750 .map_err(|_| invalid_resource("dynamic pool maintenance authority is poisoned"))
751 })
752 .collect::<Result<Vec<_>, _>>()?;
753
754 let mut planned = Vec::with_capacity(intents.len());
755 let mut pending = Vec::with_capacity(intents.len());
756 for (intent, pool) in intents.iter().zip(&pools) {
757 let requested_bytes = {
758 let state = pool
759 .state
760 .lock()
761 .map_err(|_| invalid_resource("dynamic backing pool is poisoned"))?;
762 if state.poisoned {
763 return Err(invalid_resource("dynamic backing pool is fail-closed"));
764 }
765 match intent {
766 DynamicPoolGrowthIntent::Additional(request) => request.requested_bytes(),
767 DynamicPoolGrowthIntent::Minimum(_) => {
768 let current = state
769 .resident_bytes
770 .checked_add(state.pending_growth_bytes)
771 .ok_or_else(|| {
772 invalid_resource("dynamic pool residency overflows u64")
773 })?;
774 let minimum = pool.domain.pool.provisioning().minimum_resident_bytes();
775 if current >= minimum {
776 continue;
777 }
778 minimum - current
779 }
780 DynamicPoolGrowthIntent::RevalidatedDeferral(blocker) => {
781 if let Some(claim_bytes) = blocker.contiguous_claim_bytes_descending() {
782 let required_growth = contiguous_packing_growth_bytes(
783 &state.allocator,
784 pool.domain.pool_id(),
785 claim_bytes,
786 )?;
787 if required_growth == 0 {
788 continue;
789 }
790 required_growth
791 } else {
792 match blocker.reason() {
793 DynamicBackingDeferralReason::GrowthRequired => {
794 let required_free = blocker
795 .free_bytes()
796 .checked_add(blocker.requested_bytes())
797 .ok_or_else(|| {
798 invalid_resource(
799 "dynamic backing deferred requirement overflows u64",
800 )
801 })?;
802 if state.allocator.free_bytes >= required_free {
803 continue;
804 }
805 required_free - state.allocator.free_bytes
806 }
807 DynamicBackingDeferralReason::FragmentedContiguous => {
808 return Err(invalid_resource(
809 "fragmented contiguous blocker lost its transaction demand",
810 ));
811 }
812 }
813 }
814 }
815 DynamicPoolGrowthIntent::RevalidatedAdmissionPressure {
816 required_free_bytes,
817 ..
818 } => {
819 let logical = self.logical_admission.snapshot()?;
823 if logical.poisoned() {
824 return Err(invalid_resource("logical admission is fail-closed"));
825 }
826 let available = logical
827 .domains()
828 .iter()
829 .find(|domain| domain.domain() == pool.domain.domain_id)
830 .ok_or_else(|| invalid_resource("pool has no logical capacity domain"))?
831 .available()
832 .get();
833 let missing = required_free_bytes
834 .saturating_sub(available)
835 .max(required_free_bytes.saturating_sub(state.allocator.free_bytes));
836 if missing == 0 {
837 continue;
838 }
839 missing
840 }
841 }
842 };
843 let chunk_bytes = align_up_resource(requested_bytes, pool.allocation_quantum())?;
844 let chunk = {
845 let mut state = pool
846 .state
847 .lock()
848 .map_err(|_| invalid_resource("dynamic backing pool is poisoned"))?;
849 if state.poisoned {
850 return Err(invalid_resource("dynamic backing pool is fail-closed"));
851 }
852 let ordinal = state.next_chunk_ordinal;
853 let generation = state.next_chunk_generation;
854 let next_ordinal = ordinal
855 .checked_add(1)
856 .ok_or_else(|| invalid_resource("dynamic chunk ordinal space is exhausted"))?;
857 let next_generation = generation.checked_add(1).ok_or_else(|| {
858 invalid_resource("dynamic chunk generation space is exhausted")
859 })?;
860 let next_pending = state
861 .pending_growth_bytes
862 .checked_add(chunk_bytes)
863 .ok_or_else(|| invalid_resource("pending dynamic growth bytes overflow u64"))?;
864 state.next_chunk_ordinal = next_ordinal;
865 state.next_chunk_generation = next_generation;
866 state.pending_growth_bytes = next_pending;
867 BackingChunkIdentity::from_parts(
868 pool.domain.pool_id().clone(),
869 ordinal,
870 generation,
871 )?
872 };
873 pending.push(PendingGrowthGuard {
874 pool: Arc::clone(pool),
875 bytes: chunk_bytes,
876 armed: true,
877 });
878 let expected_resource_id = ResourceId::new(format!(
879 "{}/chunk/{}/{}",
880 pool.domain.pool_id().as_str(),
881 chunk.ordinal(),
882 chunk.generation()
883 ))?;
884 planned.push(PlannedDynamicGrowth {
885 pool: Arc::clone(pool),
886 chunk,
887 expected_resource_id,
888 chunk_bytes,
889 });
890 }
891 if planned.is_empty() {
892 return Ok(DynamicPoolGrowthBatchReceipt {
893 coordinator_id: self.logical_admission.id(),
894 growths: Vec::new(),
895 capacity_epoch: self.logical_admission.epochs()?.capacity_epoch(),
896 rebalance: None,
897 maintenance_boundary: None,
898 });
899 }
900
901 let total_bytes = planned.iter().try_fold(0_u64, |total, growth| {
902 total
903 .checked_add(growth.chunk_bytes)
904 .ok_or_else(|| invalid_resource("dynamic maintenance batch bytes overflow u64"))
905 })?;
906 let capacity_availability = self.budget.availability_snapshot()?;
907 let reservation = match DeviceCapacityReservation::reserve(&self.budget, total_bytes) {
908 Ok(reservation) => reservation,
909 Err(VNextError::DeviceCapacityUnavailable(pressure)) => {
910 *capacity_blocked = Some(DynamicDeviceCapacityBlocked {
911 pressure: pressure.clone(),
912 availability: capacity_availability,
913 planned_domains: planned
914 .iter()
915 .map(|growth| growth.pool.domain.domain_id)
916 .collect(),
917 });
918 return Err(VNextError::DeviceCapacityUnavailable(pressure));
919 }
920 Err(error) => return Err(error),
921 };
922 for growth in &planned {
927 let state = growth
928 .pool
929 .state
930 .lock()
931 .map_err(|_| invalid_resource("dynamic backing pool is poisoned"))?;
932 let next_residency = state
933 .resident_bytes
934 .checked_add(state.pending_growth_bytes)
935 .ok_or_else(|| invalid_resource("dynamic pool resident bytes overflow u64"))?;
936 if next_residency
937 > growth
938 .pool
939 .domain
940 .pool
941 .provisioning()
942 .maximum_resident_bytes()
943 {
944 return Err(VNextError::DynamicPoolResidentUnavailable(
945 DynamicPoolResidentPressure::new(
946 growth.pool.domain.pool_id().clone(),
947 growth.chunk_bytes,
948 state.resident_bytes,
949 growth
950 .pool
951 .domain
952 .pool
953 .provisioning()
954 .maximum_resident_bytes(),
955 )?,
956 ));
957 }
958 }
959 let grants = reservation.commit_split(
960 &planned
961 .iter()
962 .map(|growth| growth.chunk_bytes)
963 .collect::<Vec<_>>(),
964 )?;
965 let mut allocated = Vec::with_capacity(planned.len());
966 for (growth, grant) in planned.iter().zip(grants) {
967 let transaction_identity = ResourceTransactionIdentity {
968 pool_id: self.binding.pool_id(),
969 run_id: RunId::new(format!("dynamic-grow-{}", growth.chunk.generation()))?,
970 transaction_id: TransactionId::new(format!(
971 "dynamic-grow-{}-{}",
972 growth.chunk.ordinal(),
973 growth.chunk.generation()
974 ))?,
975 request_id: self.binding.request_id().clone(),
976 };
977 let reservation_evidence = ResourceReservation {
978 resource_id: growth.expected_resource_id.clone(),
979 request_id: self.binding.request_id().clone(),
980 owner_node_id: None,
981 size_bytes: growth.chunk_bytes,
982 alignment_bytes: growth.pool.domain.pool.compatibility().alignment_bytes(),
983 usage: growth.pool.domain.pool.compatibility().usage(),
984 element_type: growth.pool.domain.pool.compatibility().element_type(),
985 retention_policy: ResourceRetentionPolicy::Plan,
986 backing_domain_id: Some(growth.pool.domain.domain_id),
987 generation: growth.chunk.generation(),
988 };
989 let request = BufferRequest::new(
990 growth.expected_resource_id.clone(),
991 growth.chunk_bytes,
992 reservation_evidence.alignment_bytes,
993 reservation_evidence.usage,
994 reservation_evidence.element_type,
995 )?;
996 validate_runtime_descriptor_for_admission(
997 self.runtime.descriptor(),
998 &self.binding,
999 "dynamic pool batch growth preflight",
1000 )?;
1001 let buffer = self
1002 .runtime
1003 .allocate(DeviceAllocationPermit {
1004 identity: &transaction_identity,
1005 binding: &self.binding,
1006 reservation: &reservation_evidence,
1007 request: &request,
1008 seal: AllocationSeal,
1009 })
1010 .map_err(|error| {
1011 invalid_resource(format!("dynamic pool device allocation failed: {error}"))
1012 })?;
1013 let actual_descriptor = self.runtime.buffer_descriptor(&buffer);
1014 validate_runtime_descriptor_for_admission(
1015 self.runtime.descriptor(),
1016 &self.binding,
1017 "dynamic pool batch growth completion",
1018 )?;
1019 if grant.bytes() != growth.chunk_bytes {
1020 return Err(invalid_resource(
1021 "dynamic pool capacity grant differs from its chunk",
1022 ));
1023 }
1024 let backing = Arc::new(ResidentChunkBacking {
1025 buffer,
1026 _grant: grant,
1027 identity: growth.chunk.clone(),
1028 descriptor: actual_descriptor.clone(),
1029 });
1030 if !reservation_evidence.matches_descriptor(&actual_descriptor) {
1031 let mut state = growth
1032 .pool
1033 .state
1034 .lock()
1035 .unwrap_or_else(std::sync::PoisonError::into_inner);
1036 state.quarantined.push(QuarantinedDynamicChunk {
1037 backing,
1038 reason: DynamicChunkQuarantineReason::DescriptorMismatch,
1039 });
1040 return Err(invalid_resource(
1041 "dynamic chunk descriptor mismatch was quarantined without capacity publication",
1042 ));
1043 }
1044 allocated.push(AllocatedDynamicGrowth { backing });
1045 }
1046
1047 let mut states = planned
1048 .iter()
1049 .map(|growth| {
1050 growth.pool.state.lock().map_err(|_| {
1051 invalid_resource("dynamic backing pool is poisoned after allocation")
1052 })
1053 })
1054 .collect::<Result<Vec<_>, _>>()?;
1055 let mut published_totals = Vec::with_capacity(planned.len());
1056 for ((growth, allocation), state) in planned.iter().zip(&allocated).zip(&states) {
1057 if state.poisoned
1058 || state.pending_growth_bytes < growth.chunk_bytes
1059 || state.chunks.contains_key(&growth.chunk.ordinal())
1060 || state
1061 .allocator
1062 .by_offset
1063 .range((growth.chunk.ordinal(), 0)..=(growth.chunk.ordinal(), u64::MAX))
1064 .next()
1065 .is_some()
1066 || state
1067 .allocator
1068 .free_bytes
1069 .checked_add(growth.chunk_bytes)
1070 .is_none()
1071 || allocation.backing.identity != growth.chunk
1072 {
1073 return Err(invalid_resource(
1074 "dynamic batch installation preconditions changed before publication",
1075 ));
1076 }
1077 published_totals.push(
1078 state
1079 .resident_bytes
1080 .checked_add(growth.chunk_bytes)
1081 .ok_or_else(|| invalid_resource("published dynamic capacity overflows u64"))?,
1082 );
1083 }
1084 for index in 0..planned.len() {
1085 let growth = &planned[index];
1086 let state = &mut states[index];
1087 state.pending_growth_bytes -= growth.chunk_bytes;
1088 pending[index].disarm();
1089 state
1090 .allocator
1091 .insert_extent(
1092 growth.chunk.ordinal(),
1093 growth.chunk.generation(),
1094 0,
1095 growth.chunk_bytes,
1096 )
1097 .expect("validated new chunk has one disjoint free extent");
1098 state.chunks.insert(
1099 growth.chunk.ordinal(),
1100 ResidentChunkState {
1101 backing: Arc::clone(&allocated[index].backing),
1102 live_segments: 0,
1103 },
1104 );
1105 }
1106 let updates = planned
1107 .iter()
1108 .zip(&published_totals)
1109 .map(|(growth, &total)| (growth.pool.domain.domain_id, CapacityUnits::new(total)))
1110 .collect::<Vec<_>>();
1111 let epochs = match self.logical_admission.set_domain_totals(&updates) {
1112 Ok(epochs) => epochs,
1113 Err(error) => {
1114 for index in 0..planned.len() {
1115 states[index]
1116 .allocator
1117 .remove_extent(planned[index].chunk.ordinal(), 0)
1118 .expect("unpublished dynamic chunk free extent remains installed");
1119 let removed = states[index]
1120 .chunks
1121 .remove(&planned[index].chunk.ordinal())
1122 .expect("unpublished dynamic chunk remains installed");
1123 states[index].quarantined.push(QuarantinedDynamicChunk {
1124 backing: removed.backing,
1125 reason: DynamicChunkQuarantineReason::PublicationRejected,
1126 });
1127 }
1128 return Err(error);
1129 }
1130 };
1131 for (state, &published_total) in states.iter_mut().zip(&published_totals) {
1132 state.resident_bytes = published_total;
1133 }
1134 Ok(DynamicPoolGrowthBatchReceipt {
1135 coordinator_id: self.logical_admission.id(),
1136 growths: planned
1137 .iter()
1138 .zip(published_totals)
1139 .map(
1140 |(growth, published_capacity_bytes)| DynamicPoolGrowthReceipt {
1141 pool_id: growth.pool.domain.pool_id().clone(),
1142 chunk: growth.chunk.clone(),
1143 chunk_bytes: growth.chunk_bytes,
1144 published_capacity_bytes,
1145 capacity_epoch: epochs.capacity_epoch(),
1146 },
1147 )
1148 .collect(),
1149 capacity_epoch: epochs.capacity_epoch(),
1150 rebalance: None,
1151 maintenance_boundary: None,
1152 })
1153 }
1154
1155 pub(in crate::vnext::resource) fn prepare_claim(
1156 &self,
1157 requests: &[EvaluatedBackingRequest<'_>],
1158 ) -> Result<BackingPrepareDecision<R>, VNextError> {
1159 if requests.is_empty() {
1160 return Ok(BackingPrepareDecision::Prepared(
1161 PreparedBackingClaim::empty(),
1162 ));
1163 }
1164 let lifetime = requests
1165 .first()
1166 .and_then(|request| request.projections.first())
1167 .map(|projection| projection.descriptor.lifetime())
1168 .ok_or_else(|| invalid_resource("dynamic backing request has no projection"))?;
1169 self.prepare_claim_scoped(
1170 requests,
1171 DynamicBackingClaimScope::from(lifetime),
1172 DynamicBackingClaimResidency::Transient,
1173 )
1174 }
1175
1176 fn reusable_capacity_shape_for_requests(
1177 &self,
1178 requests: &[EvaluatedBackingRequest<'_>],
1179 ) -> Result<Option<DynamicResourceShape>, VNextError> {
1180 let reusable_execution_bucket_id = requests
1181 .first()
1182 .and_then(|request| request.reusable_execution_bucket_id.as_ref());
1183 if requests.iter().any(|request| {
1184 request.reusable_execution_bucket_id.as_ref() != reusable_execution_bucket_id
1185 }) {
1186 return Err(invalid_resource(
1187 "one dynamic backing claim cannot mix reusable execution buckets",
1188 ));
1189 }
1190 reusable_execution_bucket_id
1191 .map(|bucket_id| {
1192 let bucket = self
1193 .reusable_execution
1194 .as_ref()
1195 .and_then(|plan| plan.bucket(bucket_id))
1196 .map(|resolved| resolved.bucket())
1197 .ok_or_else(|| {
1198 invalid_resource(
1199 "dynamic backing claim references a reusable bucket outside its immutable plan",
1200 )
1201 })?;
1202 Ok(DynamicResourceShape::from_validated(
1203 bucket.capacity().maximum_sequences(),
1204 bucket.capacity().maximum_tokens(),
1205 bucket.capacity().maximum_pages(),
1206 ))
1207 })
1208 .transpose()
1209 }
1210
1211 pub(in crate::vnext::resource) fn prepare_lane_stable_claim(
1212 self: &Arc<Self>,
1213 lane: &Arc<ExecutionLane<R>>,
1214 requests: &[EvaluatedBackingRequest<'_>],
1215 ) -> Result<LaneBackingPrepareDecision, VNextError> {
1216 if !Arc::ptr_eq(&self.runtime, lane.runtime_arc())
1217 || lane.descriptor() != self.runtime.descriptor()
1218 || !lane.is_reusable()
1219 {
1220 return Err(invalid_resource(
1221 "lane-stable backing requires the reusable execution lane bound to this plan runtime",
1222 ));
1223 }
1224 if requests.is_empty() {
1225 return Ok(LaneBackingPrepareDecision::Prepared(
1226 PreparedLaneBackingClaim::new(Vec::new(), None)?,
1227 ));
1228 }
1229 let reusable_capacity_shape = self.reusable_capacity_shape_for_requests(requests)?;
1230 if reusable_capacity_shape.is_none() {
1231 return match self.prepare_claim(requests)? {
1232 BackingPrepareDecision::Prepared(prepared) => {
1233 Ok(LaneBackingPrepareDecision::Prepared(
1234 PreparedLaneBackingClaim::new(prepared.commit(), None)?,
1235 ))
1236 }
1237 BackingPrepareDecision::Deferred(deferred) => {
1238 Ok(LaneBackingPrepareDecision::Deferred(deferred))
1239 }
1240 };
1241 }
1242 let lifetime = requests
1243 .first()
1244 .and_then(|request| request.projections.first())
1245 .map(|projection| projection.descriptor.lifetime())
1246 .ok_or_else(|| invalid_resource("dynamic backing request has no projection"))?;
1247 if !matches!(
1248 lifetime,
1249 AllocationLifetime::Step | AllocationLifetime::Invocation
1250 ) || requests.iter().any(|request| {
1251 request.projections.is_empty()
1252 || request.projections.iter().any(|projection| {
1253 projection.descriptor.lifetime() != lifetime
1254 || projection.descriptor.initialization() != StateInitialization::None
1255 })
1256 }) {
1257 return Err(invalid_resource(
1258 "lane-stable backing accepts only non-initialized Step or Invocation resources",
1259 ));
1260 }
1261
1262 let mut canonical_requests = requests.iter().collect::<Vec<_>>();
1263 canonical_requests
1264 .sort_unstable_by(|left, right| left.claim_identity.cmp(&right.claim_identity));
1265 let key = lane_stable_layout_key(lane.id(), lifetime, &canonical_requests)?;
1266 let lane_owner: Arc<dyn LaneStableArenaLane> =
1267 Arc::clone(lane) as Arc<dyn LaneStableArenaLane>;
1268
1269 {
1270 {
1271 let mut arenas = self
1272 .lane_stable_arenas
1273 .lock()
1274 .map_err(|_| invalid_resource("lane-stable arena registry is poisoned"))?;
1275 if arenas.poisoned {
1276 return Err(invalid_resource(
1277 "lane-stable arena registry is fail-closed",
1278 ));
1279 }
1280 let now = arenas.tick();
1281 if let Some(entry) = arenas.entries.get_mut(&key) {
1282 let owner = entry.lane.upgrade().ok_or_else(|| {
1283 invalid_resource("lane-stable arena retained an expired execution lane")
1284 })?;
1285 if !Arc::ptr_eq(&owner, &lane_owner) {
1286 return Err(invalid_resource(
1287 "lane-stable arena identity aliases another execution lane",
1288 ));
1289 }
1290 if let Some((slot_id, stable, certificate, slot_domains)) =
1291 entry.claim_idle_slot(lane.id(), now, &canonical_requests)?
1292 {
1293 return Ok(LaneBackingPrepareDecision::Prepared(
1294 PreparedLaneBackingClaim::certified(
1295 stable,
1296 certificate,
1297 LaneStableArenaSlotLease {
1298 arenas: Arc::clone(&self.lane_stable_arenas),
1299 logical_admission: self.logical_admission.clone(),
1300 availability_domains: slot_domains,
1301 key: key.clone(),
1302 slot_id,
1303 },
1304 ),
1305 ));
1306 }
1307 }
1308 }
1309
1310 match self.prepare_claim_scoped(
1311 requests,
1312 DynamicBackingClaimScope::from(lifetime),
1313 DynamicBackingClaimResidency::LaneStable,
1314 )? {
1315 BackingPrepareDecision::Prepared(prepared) => {
1316 let mut arenas = self
1317 .lane_stable_arenas
1318 .lock()
1319 .map_err(|_| invalid_resource("lane-stable arena registry is poisoned"))?;
1320 if arenas.poisoned {
1321 return Err(invalid_resource(
1322 "lane-stable arena registry is fail-closed",
1323 ));
1324 }
1325 let now = arenas.tick();
1326 if let Some(entry) = arenas.entries.get_mut(&key) {
1327 let owner = entry.lane.upgrade().ok_or_else(|| {
1328 invalid_resource("lane-stable arena retained an expired execution lane")
1329 })?;
1330 if !Arc::ptr_eq(&owner, &lane_owner) {
1331 return Err(invalid_resource(
1332 "lane-stable arena identity aliases another execution lane",
1333 ));
1334 }
1335 if let Some((slot_id, stable, certificate, slot_domains)) =
1336 entry.claim_idle_slot(lane.id(), now, &canonical_requests)?
1337 {
1338 drop(arenas);
1339 drop(prepared);
1340 return Ok(LaneBackingPrepareDecision::Prepared(
1341 PreparedLaneBackingClaim::certified(
1342 stable,
1343 certificate,
1344 LaneStableArenaSlotLease {
1345 arenas: Arc::clone(&self.lane_stable_arenas),
1346 logical_admission: self.logical_admission.clone(),
1347 availability_domains: slot_domains,
1348 key: key.clone(),
1349 slot_id,
1350 },
1351 ),
1352 ));
1353 }
1354 }
1355 let authorities = prepared.commit();
1356 let projection_bindings =
1357 bind_lane_stable_slot_projections(&authorities, &canonical_requests)?;
1358 let certificate = Arc::new(BackingClaimCertificate::from_slices(&authorities)?);
1359 let stable = authorities
1360 .iter()
1361 .map(|authority| authority.retained_for_lane(lane.id()))
1362 .collect();
1363 let availability_domains = requests
1364 .iter()
1365 .map(|request| request.domain.domain_id())
1366 .collect::<std::collections::BTreeSet<_>>()
1367 .into_iter()
1368 .collect::<Vec<_>>();
1369 let slot_id = arenas.issue_slot_id()?;
1370 let entry =
1371 arenas
1372 .entries
1373 .entry(key.clone())
1374 .or_insert_with(|| LaneStableArenaEntry {
1375 lane: Arc::downgrade(&lane_owner),
1376 slots: BTreeMap::new(),
1377 });
1378 if entry
1379 .slots
1380 .insert(
1381 slot_id,
1382 LaneStableArenaSlot {
1383 slot_id,
1384 authorities,
1385 certificate: Arc::clone(&certificate),
1386 projection_bindings,
1387 availability_domains: availability_domains.clone(),
1388 in_use: true,
1389 last_used: now,
1390 },
1391 )
1392 .is_some()
1393 {
1394 arenas.poisoned = true;
1395 return Err(invalid_resource(
1396 "lane-stable arena slot publication replaced an existing slot",
1397 ));
1398 }
1399 return Ok(LaneBackingPrepareDecision::Prepared(
1400 PreparedLaneBackingClaim::certified(
1401 stable,
1402 certificate,
1403 LaneStableArenaSlotLease {
1404 arenas: Arc::clone(&self.lane_stable_arenas),
1405 logical_admission: self.logical_admission.clone(),
1406 availability_domains: availability_domains.clone(),
1407 key: key.clone(),
1408 slot_id,
1409 },
1410 ),
1411 ));
1412 }
1413 BackingPrepareDecision::Deferred(deferred) => {
1414 let mut arenas = self
1415 .lane_stable_arenas
1416 .lock()
1417 .map_err(|_| invalid_resource("lane-stable arena registry is poisoned"))?;
1418 if arenas.poisoned {
1419 return Err(invalid_resource(
1420 "lane-stable arena registry is fail-closed",
1421 ));
1422 }
1423 let now = arenas.tick();
1424 if let Some(entry) = arenas.entries.get_mut(&key) {
1425 let owner = entry.lane.upgrade().ok_or_else(|| {
1426 invalid_resource("lane-stable arena retained an expired execution lane")
1427 })?;
1428 if !Arc::ptr_eq(&owner, &lane_owner) {
1429 return Err(invalid_resource(
1430 "lane-stable arena identity aliases another execution lane",
1431 ));
1432 }
1433 if let Some((slot_id, stable, certificate, slot_domains)) =
1434 entry.claim_idle_slot(lane.id(), now, &canonical_requests)?
1435 {
1436 drop(arenas);
1437 drop(deferred);
1438 return Ok(LaneBackingPrepareDecision::Prepared(
1439 PreparedLaneBackingClaim::certified(
1440 stable,
1441 certificate,
1442 LaneStableArenaSlotLease {
1443 arenas: Arc::clone(&self.lane_stable_arenas),
1444 logical_admission: self.logical_admission.clone(),
1445 availability_domains: slot_domains,
1446 key: key.clone(),
1447 slot_id,
1448 },
1449 ),
1450 ));
1451 }
1452 }
1453 return Ok(LaneBackingPrepareDecision::Deferred(deferred));
1454 }
1455 }
1456 }
1457 }
1458
1459 pub(in crate::vnext::resource) fn try_reclaim_expired_lane_slots(
1460 &self,
1461 ) -> Result<bool, VNextError> {
1462 let expired_entries = {
1463 let mut arenas = self
1464 .lane_stable_arenas
1465 .lock()
1466 .map_err(|_| invalid_resource("lane-stable arena registry is poisoned"))?;
1467 if arenas.poisoned {
1468 return Err(invalid_resource(
1469 "lane-stable arena registry is fail-closed",
1470 ));
1471 }
1472 arenas.take_expired_lanes()?
1473 };
1474 let reclaimed = !expired_entries.is_empty();
1475 drop(expired_entries);
1478 Ok(reclaimed)
1479 }
1480
1481 pub(in crate::vnext::resource) fn try_reclaim_one_idle_lane_slot(
1482 &self,
1483 ) -> Result<bool, VNextError> {
1484 if self.try_reclaim_expired_lane_slots()? {
1485 return Ok(true);
1486 }
1487 let mut candidates = {
1488 let arenas = self
1489 .lane_stable_arenas
1490 .lock()
1491 .map_err(|_| invalid_resource("lane-stable arena registry is poisoned"))?;
1492 if arenas.poisoned {
1493 return Err(invalid_resource(
1494 "lane-stable arena registry is fail-closed",
1495 ));
1496 }
1497 arenas
1498 .entries
1499 .iter()
1500 .filter_map(|(key, entry)| entry.lane.upgrade().map(|lane| (key, entry, lane)))
1501 .flat_map(|(key, entry, lane)| {
1502 entry
1503 .slots
1504 .values()
1505 .filter(|slot| !slot.in_use)
1506 .map(move |slot| LaneStableArenaEvictionCandidate {
1507 key: key.clone(),
1508 slot_id: slot.slot_id,
1509 last_used: slot.last_used,
1510 lane: Arc::clone(&lane),
1511 })
1512 })
1513 .collect::<Vec<_>>()
1514 };
1515 candidates.sort_by_key(|candidate| candidate.last_used);
1516
1517 for candidate in candidates {
1518 if !candidate.lane.try_trim_reusable_executables()? {
1519 continue;
1520 }
1521 let victim = {
1522 let mut arenas = self
1523 .lane_stable_arenas
1524 .lock()
1525 .map_err(|_| invalid_resource("lane-stable arena registry is poisoned"))?;
1526 if arenas.poisoned {
1527 return Err(invalid_resource(
1528 "lane-stable arena registry is fail-closed",
1529 ));
1530 }
1531 let removable = arenas
1532 .entries
1533 .get(&candidate.key)
1534 .and_then(|entry| entry.slots.get(&candidate.slot_id))
1535 .is_some_and(|slot| !slot.in_use && !slot.has_external_address_pins());
1536 if !removable {
1537 None
1538 } else {
1539 let (victim, remove_entry) = {
1540 let entry = arenas.entries.get_mut(&candidate.key).ok_or_else(|| {
1541 invalid_resource("lane-stable arena eviction lost its entry")
1542 })?;
1543 let victim = entry.slots.remove(&candidate.slot_id).ok_or_else(|| {
1544 invalid_resource("lane-stable arena eviction lost its idle slot")
1545 })?;
1546 (victim, entry.slots.is_empty())
1547 };
1548 if remove_entry {
1549 arenas.entries.remove(&candidate.key);
1550 }
1551 Some(victim)
1552 }
1553 };
1554 if let Some(victim) = victim {
1555 drop(victim);
1556 return Ok(true);
1557 }
1558 }
1559 Ok(false)
1560 }
1561
1562 pub(in crate::vnext::resource) fn prepare_initial_sequence_claim(
1563 &self,
1564 requests: &[EvaluatedBackingRequest<'_>],
1565 ) -> Result<BackingPrepareDecision<R>, VNextError> {
1566 self.prepare_claim_scoped(
1567 requests,
1568 DynamicBackingClaimScope::InitialSequenceBundle,
1569 DynamicBackingClaimResidency::Transient,
1570 )
1571 }
1572
1573 pub(in crate::vnext::resource) fn maintain_checkpoint_capacity(
1577 &self,
1578 requests: &[EvaluatedBackingRequest<'_>],
1579 ) -> Result<DynamicPoolGrowthBatchReceipt, VNextError> {
1580 self.maintain_checkpoint_capacity_observed(requests, &mut None)
1581 }
1582
1583 pub(in crate::vnext::resource) fn maintain_checkpoint_capacity_with_idle_reclaim(
1584 &self,
1585 requests: &[EvaluatedBackingRequest<'_>],
1586 ) -> Result<DynamicPoolGrowthBatchReceipt, VNextError> {
1587 let mut capacity_blocked = None;
1588 match self.maintain_checkpoint_capacity_observed(requests, &mut capacity_blocked) {
1589 Err(VNextError::DeviceCapacityUnavailable(pressure)) => {
1590 let blocked = capacity_blocked.as_ref().ok_or_else(|| {
1591 invalid_resource("checkpoint growth lost its device capacity observation")
1592 })?;
1593 let targets = requests
1597 .iter()
1598 .map(|request| request.domain.domain_id)
1599 .collect::<std::collections::BTreeSet<_>>()
1600 .into_iter()
1601 .collect::<Vec<_>>();
1602 let attempt = self.reclaim_idle_chunks(
1603 &pressure,
1604 blocked.availability,
1605 &targets,
1606 &CapacityVector::empty(),
1607 &[],
1608 ReclaimTargetPolicy::PreserveAll,
1609 )?;
1610 tracing::debug!(
1611 coordinator_id = ?attempt.boundary.coordinator_id(),
1612 ?pressure,
1613 protected_target_domains = ?targets,
1614 reclaim_sufficient = attempt.boundary.reclaim_sufficient(),
1615 candidate_chunks = attempt.boundary.reclaim_candidate_chunks(),
1616 candidate_bytes = attempt.boundary.reclaim_candidate_bytes(),
1617 selected_bytes = attempt.boundary.selected_bytes(),
1618 reclaimed_bytes = attempt.rebalance.as_ref().map_or(0, |r| r.reclaimed_bytes()),
1619 reclaimed_chunks = attempt.rebalance.as_ref().map_or(0, |r| r.reclaimed_chunks()),
1620 "optional checkpoint idle reclaim evaluated"
1621 );
1622 if tracing::enabled!(tracing::Level::DEBUG) {
1623 for pool in attempt.boundary.pools() {
1624 let reclaimed_bytes = attempt
1625 .rebalance
1626 .as_ref()
1627 .and_then(|rebalance| {
1628 rebalance
1629 .pools()
1630 .iter()
1631 .find(|reclaimed| reclaimed.pool_id() == pool.pool_id())
1632 })
1633 .map_or(0, |reclaimed| reclaimed.reclaimed_bytes());
1634 tracing::debug!(
1635 coordinator_id = ?attempt.boundary.coordinator_id(),
1636 domain = ?pool.domain_id(),
1637 pool_id = pool.pool_id().as_str(),
1638 excluded = pool.excluded_from_reclaim(),
1639 resident_bytes = pool.resident_bytes(),
1640 free_bytes = pool.free_bytes(),
1641 logical_used_bytes = pool.logical_used_bytes(),
1642 physical_live_bytes = pool.live_occupancy().total().physical_bytes(),
1643 pending_growth_bytes = pool.pending_growth_bytes(),
1644 resident_floor_bytes = pool.resident_floor_bytes(),
1645 bytes_above_floor = pool.reclaimable_bytes(),
1646 reclaimed_bytes,
1647 "optional checkpoint idle reclaim pool boundary"
1648 );
1649 }
1650 }
1651 let Some(rebalance) = attempt.rebalance else {
1652 return Err(VNextError::DeviceCapacityUnavailable(pressure));
1653 };
1654 let mut receipt = self.maintain_checkpoint_capacity(requests)?;
1658 tracing::debug!(
1659 coordinator_id = ?receipt.coordinator_id(),
1660 reclaimed_bytes = rebalance.reclaimed_bytes(),
1661 growths = ?receipt.growths(),
1662 "optional checkpoint idle reclaim followed by fresh growth"
1663 );
1664 receipt.rebalance = Some(rebalance);
1665 receipt.maintenance_boundary = Some(attempt.boundary);
1666 Ok(receipt)
1667 }
1668 result => result,
1669 }
1670 }
1671
1672 fn maintain_checkpoint_capacity_observed(
1673 &self,
1674 requests: &[EvaluatedBackingRequest<'_>],
1675 capacity_blocked: &mut Option<DynamicDeviceCapacityBlocked>,
1676 ) -> Result<DynamicPoolGrowthBatchReceipt, VNextError> {
1677 self.validate_checkpoint_requests(requests)?;
1678 let mut by_pool = BTreeMap::<DynamicBackingPoolId, Vec<u64>>::new();
1679 for request in requests {
1680 by_pool
1681 .entry(request.domain.pool_id().clone())
1682 .or_default()
1683 .push(request.capacity_size_bytes);
1684 }
1685 let mut intents = Vec::with_capacity(by_pool.len());
1686 for (pool_id, mut claims) in by_pool {
1687 claims.sort_unstable_by(|left, right| right.cmp(left));
1688 let pool = self
1689 .pools
1690 .get(&pool_id)
1691 .ok_or_else(|| invalid_resource("checkpoint pool is absent"))?;
1692 let state = pool
1693 .state
1694 .lock()
1695 .map_err(|_| invalid_resource("checkpoint pool is poisoned"))?;
1696 if state.poisoned {
1697 return Err(invalid_resource("checkpoint pool is fail-closed"));
1698 }
1699 let (growth, contiguous_claims) =
1700 match pool.domain.pool.compatibility().profile().view() {
1701 DynamicStorageView::Contiguous => (
1702 contiguous_packing_growth_bytes(&state.allocator, &pool_id, &claims)?,
1703 Some(claims),
1704 ),
1705 DynamicStorageView::PagedRegions { .. } => (
1706 claims
1707 .iter()
1708 .try_fold(0_u64, |total, bytes| total.checked_add(*bytes))
1709 .ok_or_else(|| {
1710 invalid_resource("checkpoint pool demand overflows u64")
1711 })?
1712 .saturating_sub(state.allocator.free_bytes),
1713 None,
1714 ),
1715 };
1716 if growth != 0 {
1717 intents.push(DynamicPoolGrowthIntent::RevalidatedDeferral(
1718 DynamicBackingBlocker {
1719 pool_id,
1720 domain_id: pool.domain.domain_id,
1721 reason: DynamicBackingDeferralReason::GrowthRequired,
1722 requested_bytes: growth,
1723 free_bytes: state.allocator.free_bytes,
1724 largest_contiguous_bytes: state.allocator.largest_contiguous_bytes(),
1725 free_extent_layout_fingerprint: free_extent_layout_fingerprint(
1726 &state.allocator,
1727 ),
1728 contiguous_claim_bytes_descending: contiguous_claims,
1729 },
1730 ));
1731 }
1732 }
1733 self.maintain_pools_observed(intents, capacity_blocked)
1736 }
1737
1738 pub(in crate::vnext::resource) fn prepare_checkpoint_claim(
1739 &self,
1740 requests: &[EvaluatedBackingRequest<'_>],
1741 ) -> Result<BackingPrepareDecision<R>, VNextError> {
1742 self.validate_checkpoint_requests(requests)?;
1743 self.prepare_claim_scoped(
1744 requests,
1745 DynamicBackingClaimScope::Checkpoint,
1746 DynamicBackingClaimResidency::Transient,
1747 )
1748 }
1749
1750 fn validate_checkpoint_requests(
1751 &self,
1752 requests: &[EvaluatedBackingRequest<'_>],
1753 ) -> Result<(), VNextError> {
1754 if requests.is_empty()
1755 || requests.iter().any(|request| {
1756 !self
1757 .domains
1758 .iter()
1759 .any(|domain| std::ptr::eq(domain, request.domain))
1760 || request.reusable_execution_bucket_id.is_some()
1761 || request.projections.len() != 1
1762 || request.projections.iter().any(|projection| {
1763 projection.descriptor.lifetime() != AllocationLifetime::Sequence
1764 || projection.descriptor.usage() != super::BufferUsage::State
1765 || *projection.descriptor.kind() != super::AllocationKind::Value
1766 })
1767 })
1768 {
1769 return Err(invalid_resource(
1770 "checkpoint backing requires this plan's non-empty Sequence state projections",
1771 ));
1772 }
1773 Ok(())
1774 }
1775
1776 fn prepare_claim_scoped(
1777 &self,
1778 requests: &[EvaluatedBackingRequest<'_>],
1779 scope: DynamicBackingClaimScope,
1780 residency: DynamicBackingClaimResidency,
1781 ) -> Result<BackingPrepareDecision<R>, VNextError> {
1782 if requests.is_empty() {
1783 return Ok(BackingPrepareDecision::Prepared(
1784 PreparedBackingClaim::empty(),
1785 ));
1786 }
1787 let reusable_capacity_shape = self.reusable_capacity_shape_for_requests(requests)?;
1788 let mut grouped =
1789 BTreeMap::<DynamicBackingPoolId, Vec<&EvaluatedBackingRequest<'_>>>::new();
1790 for request in requests {
1791 grouped
1792 .entry(request.domain.pool_id().clone())
1793 .or_default()
1794 .push(request);
1795 }
1796 let mut groups = Vec::with_capacity(grouped.len());
1797 for (pool_id, mut requests) in grouped {
1798 requests.sort_by(|left, right| left.claim_identity.cmp(&right.claim_identity));
1799 if requests
1800 .windows(2)
1801 .any(|pair| pair[0].claim_identity == pair[1].claim_identity)
1802 {
1803 return Err(invalid_resource(
1804 "dynamic backing reservation contains a duplicate physical claim",
1805 ));
1806 }
1807 let pool = self.pools.get(&pool_id).cloned().ok_or_else(|| {
1808 invalid_resource("dynamic backing reservation references an unknown pool")
1809 })?;
1810 groups.push((pool, requests));
1811 }
1812 let protected_packing_envelopes = groups
1813 .iter()
1814 .map(|(pool, requests)| {
1815 DynamicBackingPackingEnvelope::new(
1816 pool.domain.pool_id().clone(),
1817 pool.domain.domain_id,
1818 requests
1819 .iter()
1820 .map(|request| request.capacity_size_bytes)
1821 .collect(),
1822 )
1823 })
1824 .collect::<Result<Vec<_>, VNextError>>()?;
1825 let protected_immediate = CapacityVector::new(
1826 protected_packing_envelopes
1827 .iter()
1828 .map(|envelope| {
1829 CapacityEntry::new(
1830 envelope.domain_id(),
1831 CapacityUnits::new(envelope.total_bytes()?),
1832 )
1833 })
1834 .collect::<Result<Vec<_>, VNextError>>()?,
1835 )?;
1836 'prepare: loop {
1837 let mut states = groups
1838 .iter()
1839 .map(|(pool, _)| {
1840 pool.state
1841 .lock()
1842 .map_err(|_| invalid_resource("dynamic backing pool is poisoned"))
1843 })
1844 .collect::<Result<Vec<_>, _>>()?;
1845 for (group_index, (pool, pool_requests)) in groups.iter().enumerate() {
1846 if states[group_index].poisoned {
1847 return Err(invalid_resource("dynamic backing pool is fail-closed"));
1848 }
1849 let quantum = pool.allocation_quantum();
1850 for request in pool_requests {
1851 let projection_ids = request
1852 .projections
1853 .iter()
1854 .map(|projection| projection.descriptor.base_resource_id().clone())
1855 .collect::<Vec<_>>();
1856 let single_projection = request.projections.len() == 1
1857 && request.projections[0].physical_offset_bytes == 0
1858 && request.projections[0].capacity_size_bytes
1859 == request.capacity_size_bytes;
1860 let shared_step_slot = request.projections.len() > 1
1861 && request
1862 .projections
1863 .iter()
1864 .all(|projection| projection.physical_offset_bytes == 0)
1865 && request
1866 .projections
1867 .iter()
1868 .map(|projection| projection.capacity_size_bytes)
1869 .max()
1870 == Some(request.capacity_size_bytes)
1871 && pool.domain.pool.step_resource_slots().iter().any(|slot| {
1872 slot.kind() == StepResourceSlotKind::OrderedSingleFenceStepWave
1873 && slot.resource_ids() == request.claim_identity.resource_ids()
1874 });
1875 let invocation_wave =
1876 self.validate_invocation_wave_projection(pool, request)?;
1877 if request.domain.pool_id() != pool.domain.pool_id()
1878 || request.claim_identity.pool_id() != pool.domain.pool_id()
1879 || request.claim_identity.resource_ids() != projection_ids
1880 || request.projections.is_empty()
1881 || request.projections.windows(2).any(|pair| {
1882 pair[0].descriptor.base_resource_id()
1883 >= pair[1].descriptor.base_resource_id()
1884 })
1885 || request.projections.iter().any(|projection| {
1886 let capacity_matches_plan = match reusable_capacity_shape {
1887 Some(shape) => {
1888 matches!(
1889 projection.descriptor.lifetime(),
1890 AllocationLifetime::Step | AllocationLifetime::Invocation
1891 ) && projection
1892 .descriptor
1893 .evaluate_request_bytes_for_shape(shape)
1894 .is_ok_and(|bytes| bytes == projection.capacity_size_bytes)
1895 }
1896 None => {
1897 scope == DynamicBackingClaimScope::Checkpoint
1898 || projection.logical_size_bytes
1899 == projection.capacity_size_bytes
1900 }
1901 };
1902 projection.descriptor.pool_id() != pool.domain.pool_id()
1903 || !scope.accepts(projection.descriptor.lifetime())
1904 || !capacity_matches_plan
1905 || projection.logical_size_bytes == 0
1906 || projection.logical_size_bytes > projection.capacity_size_bytes
1907 || projection.capacity_size_bytes == 0
1908 || projection.capacity_size_bytes % quantum != 0
1909 || projection.physical_offset_bytes % quantum != 0
1910 || projection
1911 .physical_offset_bytes
1912 .checked_add(projection.capacity_size_bytes)
1913 .is_none_or(|end| end > request.capacity_size_bytes)
1914 || !request
1915 .domain
1916 .descriptors
1917 .iter()
1918 .any(|descriptor| descriptor == projection.descriptor)
1919 })
1920 || request.capacity_size_bytes == 0
1921 || request.capacity_size_bytes % quantum != 0
1922 || !(single_projection || shared_step_slot || invocation_wave)
1923 {
1924 return Err(invalid_resource(
1925 "dynamic backing request violates its physical claim, projection, pool, or allocation quantum",
1926 ));
1927 }
1928 }
1929 }
1930 let blockers = groups
1931 .iter()
1932 .enumerate()
1933 .map(|(group_index, (pool, pool_requests))| {
1934 let requested_group_bytes =
1935 pool_requests.iter().try_fold(0_u64, |total, request| {
1936 total
1937 .checked_add(request.capacity_size_bytes)
1938 .ok_or_else(|| {
1939 invalid_resource("dynamic backing batch bytes overflow u64")
1940 })
1941 })?;
1942 let state = &states[group_index];
1943 if state.allocator.free_bytes >= requested_group_bytes {
1944 return Ok(None);
1945 }
1946 let (reason, requested_bytes, contiguous_claim_bytes_descending) =
1947 match pool.domain.pool.compatibility().profile().view() {
1948 DynamicStorageView::Contiguous => {
1949 let mut claim_bytes = pool_requests
1950 .iter()
1951 .map(|request| request.capacity_size_bytes)
1952 .collect::<Vec<_>>();
1953 claim_bytes.sort_unstable_by(|left, right| right.cmp(left));
1954 let growth = contiguous_packing_growth_bytes(
1955 &state.allocator,
1956 pool.domain.pool_id(),
1957 &claim_bytes,
1958 )?;
1959 if growth == 0 {
1960 return Err(invalid_resource(
1961 "insufficient contiguous capacity produced zero packing growth",
1962 ));
1963 }
1964 (
1965 DynamicBackingDeferralReason::GrowthRequired,
1966 growth,
1967 Some(claim_bytes),
1968 )
1969 }
1970 DynamicStorageView::PagedRegions { .. } => (
1971 DynamicBackingDeferralReason::GrowthRequired,
1972 requested_group_bytes - state.allocator.free_bytes,
1973 None,
1974 ),
1975 };
1976 Ok(Some(DynamicBackingBlocker {
1977 pool_id: pool.domain.pool_id().clone(),
1978 domain_id: pool.domain.domain_id,
1979 reason,
1980 requested_bytes,
1981 free_bytes: state.allocator.free_bytes,
1982 largest_contiguous_bytes: state.allocator.largest_contiguous_bytes(),
1983 free_extent_layout_fingerprint: free_extent_layout_fingerprint(
1984 &state.allocator,
1985 ),
1986 contiguous_claim_bytes_descending,
1987 }))
1988 })
1989 .collect::<Result<Vec<_>, VNextError>>()?
1990 .into_iter()
1991 .flatten()
1992 .collect::<Vec<_>>();
1993 if !blockers.is_empty() {
1994 drop(states);
1995 if let Some(deferred) = self.confirm_backing_deferral(
1996 blockers,
1997 scope,
1998 protected_immediate.clone(),
1999 protected_packing_envelopes.clone(),
2000 )? {
2001 return Ok(BackingPrepareDecision::Deferred(deferred));
2002 }
2003 continue 'prepare;
2004 }
2005 let segment_generations = groups
2006 .iter()
2007 .map(|(pool, requests)| {
2008 (0..requests.len())
2009 .map(|_| {
2010 pool.next_extent_generation
2011 .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| {
2012 current.checked_add(1)
2013 })
2014 .map_err(|_| {
2015 invalid_resource("dynamic extent generation space is exhausted")
2016 })
2017 })
2018 .collect::<Result<Vec<_>, _>>()
2019 })
2020 .collect::<Result<Vec<_>, _>>()?;
2021 let mut selections = groups
2022 .iter()
2023 .map(|_| {
2024 Vec::<(
2025 &EvaluatedBackingRequest<'_>,
2026 u64,
2027 Vec<BackingSegment>,
2028 DynamicBackingClaimOccupancy,
2029 )>::new()
2030 })
2031 .collect::<Vec<_>>();
2032 let mut journals = groups
2033 .iter()
2034 .map(|_| Vec::<Vec<BackingSegment>>::new())
2035 .collect::<Vec<_>>();
2036 for group_index in 0..groups.len() {
2037 let (pool, pool_requests) = &groups[group_index];
2038 let profile = pool.domain.pool.compatibility().profile();
2039 let mut allocation_requests = pool_requests.clone();
2040 allocation_requests.sort_by(|left, right| {
2041 right
2042 .capacity_size_bytes
2043 .cmp(&left.capacity_size_bytes)
2044 .then_with(|| left.claim_identity.cmp(&right.claim_identity))
2045 });
2046 for request in allocation_requests {
2047 let reserved = match match profile.view() {
2048 DynamicStorageView::Contiguous => states[group_index]
2049 .allocator
2050 .allocate_contiguous(pool.domain.pool_id(), request.capacity_size_bytes)
2051 .map(|segment| segment.map(|segment| vec![segment])),
2052 DynamicStorageView::PagedRegions { block_bytes } => {
2053 states[group_index].allocator.allocate_paged(
2054 pool.domain.pool_id(),
2055 request.capacity_size_bytes,
2056 block_bytes,
2057 )
2058 }
2059 } {
2060 Ok(reserved) => reserved,
2061 Err(error) => {
2062 states[group_index].poisoned = true;
2063 rollback_free_extent_journal(&mut states, &journals)?;
2064 return Err(error);
2065 }
2066 };
2067 let Some(segments) = reserved else {
2068 rollback_free_extent_journal(&mut states, &journals)?;
2069 if !matches!(profile.view(), DynamicStorageView::Contiguous) {
2070 states[group_index].poisoned = true;
2071 return Err(invalid_resource(
2072 "paged backing allocation failed after its aggregate fit check",
2073 ));
2074 }
2075 let free_bytes = states[group_index].allocator.free_bytes;
2076 let reason = DynamicBackingDeferralReason::FragmentedContiguous;
2077 let mut claim_bytes_descending = pool_requests
2078 .iter()
2079 .map(|request| request.capacity_size_bytes)
2080 .collect::<Vec<_>>();
2081 claim_bytes_descending.sort_unstable_by(|left, right| right.cmp(left));
2082 let requested_bytes = contiguous_packing_growth_bytes(
2083 &states[group_index].allocator,
2084 pool.domain.pool_id(),
2085 &claim_bytes_descending,
2086 )?;
2087 if requested_bytes == 0 {
2088 return Err(invalid_resource(
2089 "contiguous packing failed without a progress-producing growth",
2090 ));
2091 }
2092 let blocker = DynamicBackingBlocker {
2093 pool_id: pool.domain.pool_id().clone(),
2094 domain_id: pool.domain.domain_id,
2095 reason,
2096 requested_bytes,
2097 free_bytes,
2098 largest_contiguous_bytes: states[group_index]
2099 .allocator
2100 .largest_contiguous_bytes(),
2101 free_extent_layout_fingerprint: free_extent_layout_fingerprint(
2102 &states[group_index].allocator,
2103 ),
2104 contiguous_claim_bytes_descending: Some(claim_bytes_descending),
2105 };
2106 drop(states);
2107 if let Some(deferred) = self.confirm_backing_deferral(
2108 vec![blocker],
2109 scope,
2110 protected_immediate.clone(),
2111 protected_packing_envelopes.clone(),
2112 )? {
2113 return Ok(BackingPrepareDecision::Deferred(deferred));
2114 }
2115 continue 'prepare;
2116 };
2117 journals[group_index].push(segments.clone());
2118 let extent_bytes = match segments.iter().try_fold(0_u64, |total, segment| {
2119 total.checked_add(segment.length_bytes()).ok_or_else(|| {
2120 invalid_resource("dynamic backing extent bytes overflow u64")
2121 })
2122 }) {
2123 Ok(bytes) => bytes,
2124 Err(error) => {
2125 rollback_free_extent_journal(&mut states, &journals)?;
2126 return Err(error);
2127 }
2128 };
2129 if extent_bytes != request.capacity_size_bytes {
2130 rollback_free_extent_journal(&mut states, &journals)?;
2131 return Err(invalid_resource(
2132 "dynamic backing extents differ from their physical capacity claim",
2133 ));
2134 }
2135 let generation =
2136 segment_generations[group_index][selections[group_index].len()];
2137 let segment_count = match u64::try_from(segments.len()) {
2138 Ok(count) => count,
2139 Err(_) => {
2140 rollback_free_extent_journal(&mut states, &journals)?;
2141 return Err(invalid_resource(
2142 "dynamic backing segment count exceeds portable range",
2143 ));
2144 }
2145 };
2146 selections[group_index].push((
2147 request,
2148 generation,
2149 segments,
2150 DynamicBackingClaimOccupancy {
2151 scope,
2152 residency,
2153 physical_bytes: request.capacity_size_bytes,
2154 segment_count,
2155 },
2156 ));
2157 }
2158 }
2159
2160 for group_selections in &selections {
2161 for (request, _, segments, _) in group_selections {
2162 for projection in &request.projections {
2163 if let Err(error) = backing_segment_range(
2164 segments,
2165 projection.physical_offset_bytes,
2166 projection.capacity_size_bytes,
2167 ) {
2168 rollback_free_extent_journal(&mut states, &journals)?;
2169 return Err(error);
2170 }
2171 }
2172 }
2173 }
2174
2175 let accounting_updates = match (0..groups.len())
2176 .map(|group_index| {
2177 let mut increments = BTreeMap::<u32, u64>::new();
2178 for (_, _, segments, _) in &selections[group_index] {
2179 for segment in segments {
2180 let count = increments.entry(segment.chunk_ordinal()).or_default();
2181 *count = count.checked_add(1).ok_or_else(|| {
2182 invalid_resource(
2183 "dynamic chunk live extent increment overflows u64",
2184 )
2185 })?;
2186 }
2187 }
2188 for (&ordinal, &increment) in &increments {
2189 states[group_index]
2190 .chunks
2191 .get(&ordinal)
2192 .ok_or_else(|| invalid_resource("reserved dynamic chunk disappeared"))?
2193 .live_segments
2194 .checked_add(increment)
2195 .ok_or_else(|| {
2196 invalid_resource("dynamic chunk live extent count overflowed")
2197 })?;
2198 }
2199 let occupancy = selections[group_index].iter().try_fold(
2200 states[group_index].live_occupancy,
2201 |occupancy, (_, _, _, claim)| occupancy.checked_with_claim(*claim),
2202 )?;
2203 Ok((increments, occupancy))
2204 })
2205 .collect::<Result<Vec<_>, VNextError>>()
2206 {
2207 Ok(updates) => updates,
2208 Err(error) => {
2209 rollback_free_extent_journal(&mut states, &journals)?;
2210 return Err(error);
2211 }
2212 };
2213 for (group_index, (increments, occupancy)) in accounting_updates.into_iter().enumerate()
2214 {
2215 for (ordinal, increment) in increments {
2216 states[group_index]
2217 .chunks
2218 .get_mut(&ordinal)
2219 .expect("validated reserved dynamic chunk remains installed")
2220 .live_segments += increment;
2221 }
2222 states[group_index].live_occupancy = occupancy;
2223 }
2224 drop(states);
2225 let mut extents = Vec::new();
2226 for ((pool, _), selections) in groups.into_iter().zip(selections) {
2227 for (request, segment_generation, segments, occupancy) in selections {
2228 let projections = request
2229 .projections
2230 .iter()
2231 .map(|projection| {
2232 let mut allocation = LogicalBackingSliceAllocationEvidence {
2233 domain_id: pool.domain.domain_id,
2234 pool_id: pool.domain.pool_id().clone(),
2235 resource_id: projection.descriptor.base_resource_id().clone(),
2236 pool_instance_id: pool.instance_id,
2237 physical_claim_identity: request.claim_identity.clone(),
2238 reusable_execution_bucket_id: request
2239 .reusable_execution_bucket_id
2240 .clone(),
2241 segment_generation,
2242 segments: backing_segment_range(
2243 &segments,
2244 projection.physical_offset_bytes,
2245 projection.capacity_size_bytes,
2246 )?,
2247 physical_offset_bytes: projection.physical_offset_bytes,
2248 capacity_size_bytes: projection.capacity_size_bytes,
2249 physical_size_bytes: request.capacity_size_bytes,
2250 alignment_bytes: projection.descriptor.alignment_bytes(),
2251 usage: projection.descriptor.usage(),
2252 element_type: projection.descriptor.element_type(),
2253 storage_profile: pool.domain.pool.compatibility().profile(),
2254 initialization: projection.descriptor.initialization(),
2255 fingerprint: String::new(),
2256 };
2257 let bytes = serde_json::to_vec(&allocation).map_err(|error| {
2258 invalid_resource(format!(
2259 "logical backing allocation evidence encode failed: {error}"
2260 ))
2261 })?;
2262 allocation.fingerprint = format!("sha256/{:x}", Sha256::digest(bytes));
2263 Ok(LogicalBackingSliceEvidence {
2264 allocation: Arc::new(allocation),
2265 logical_size_bytes: projection.logical_size_bytes,
2266 })
2267 })
2268 .collect::<Result<Vec<_>, VNextError>>()?;
2269 extents.push(PreparedBackingExtent {
2270 pool: Arc::clone(&pool),
2271 claim_identity: request.claim_identity.clone(),
2272 segment_generation,
2273 occupancy,
2274 segments,
2275 capacity_size_bytes: request.capacity_size_bytes,
2276 projections,
2277 });
2278 }
2279 }
2280 return Ok(BackingPrepareDecision::Prepared(PreparedBackingClaim {
2281 extents,
2282 committed: false,
2283 }));
2284 }
2285 }
2286
2287 fn confirm_backing_deferral(
2299 &self,
2300 blockers: Vec<DynamicBackingBlocker>,
2301 scope: DynamicBackingClaimScope,
2302 protected_immediate: CapacityVector,
2303 protected_packing_envelopes: Vec<DynamicBackingPackingEnvelope>,
2304 ) -> Result<Option<DynamicBackingDeferred>, VNextError> {
2305 let wait_snapshot = self
2306 .logical_admission
2307 .wait_snapshot_for_domains(blockers.iter().map(DynamicBackingBlocker::domain_id))?;
2308 for blocker in &blockers {
2309 let pool = self.pools.get(blocker.pool_id()).ok_or_else(|| {
2310 invalid_resource("dynamic backing blocker references an unknown pool")
2311 })?;
2312 let state = pool
2313 .state
2314 .lock()
2315 .map_err(|_| invalid_resource("dynamic backing pool is poisoned"))?;
2316 if state.poisoned {
2317 return Err(invalid_resource("dynamic backing pool is fail-closed"));
2318 }
2319 if state.allocator.free_bytes != blocker.free_bytes()
2320 || state.allocator.largest_contiguous_bytes() != blocker.largest_contiguous_bytes()
2321 || free_extent_layout_fingerprint(&state.allocator)
2322 != blocker.free_extent_layout_fingerprint()
2323 {
2324 return Ok(None);
2325 }
2326 }
2327 Ok(Some(DynamicBackingDeferred {
2328 blockers,
2329 epochs: wait_snapshot.epochs(),
2330 wait_condition: wait_snapshot.wait_condition().clone(),
2331 scope,
2332 protected_immediate,
2333 protected_packing_envelopes,
2334 }))
2335 }
2336
2337 fn validate_invocation_wave_projection(
2338 &self,
2339 pool: &DynamicBackingPool<R>,
2340 request: &EvaluatedBackingRequest<'_>,
2341 ) -> Result<bool, VNextError> {
2342 let mode = pool.domain.pool.invocation_liveness_mode();
2343 if mode == InvocationLivenessMode::NoInvocationResources
2344 || request.projections.len() < 2
2345 || request.projections.iter().any(|projection| {
2346 projection.descriptor.lifetime() != super::AllocationLifetime::Invocation
2347 })
2348 {
2349 return Ok(false);
2350 }
2351 let mut expected_resources = pool
2352 .domain
2353 .pool
2354 .invocation_liveness()
2355 .iter()
2356 .flat_map(|row| row.resource_ids().iter().cloned())
2357 .collect::<Vec<_>>();
2358 expected_resources.sort();
2359 if expected_resources.windows(2).any(|pair| pair[0] == pair[1])
2360 || expected_resources != request.claim_identity.resource_ids()
2361 {
2362 return Ok(false);
2363 }
2364 let projections = request
2365 .projections
2366 .iter()
2367 .map(|projection| (projection.descriptor.base_resource_id().clone(), projection))
2368 .collect::<BTreeMap<_, _>>();
2369 if projections.len() != request.projections.len() {
2370 return Ok(false);
2371 }
2372 let rows_by_node = pool
2373 .domain
2374 .pool
2375 .invocation_liveness()
2376 .iter()
2377 .map(|row| (row.node_id(), row))
2378 .collect::<BTreeMap<_, _>>();
2379 let rows = self
2380 .nodes
2381 .iter()
2382 .filter_map(|node| rows_by_node.get(node.id()).copied())
2383 .collect::<Vec<_>>();
2384 if rows.len() != rows_by_node.len() {
2385 return Ok(false);
2386 }
2387
2388 let mut concurrent_cursor = 0_u64;
2389 let mut peak = 0_u64;
2390 for row in rows {
2391 let row_base = match mode {
2392 InvocationLivenessMode::TotalOrderReuse => 0,
2393 InvocationLivenessMode::ConservativeConcurrent => concurrent_cursor,
2394 InvocationLivenessMode::NoInvocationResources => unreachable!(),
2395 };
2396 let mut row_cursor = 0_u64;
2397 for resource_id in row.resource_ids() {
2398 let Some(projection) = projections.get(resource_id) else {
2399 return Ok(false);
2400 };
2401 let expected_offset = row_base.checked_add(row_cursor).ok_or_else(|| {
2402 invalid_resource("invocation wave projection offset overflows u64")
2403 })?;
2404 if projection.physical_offset_bytes != expected_offset {
2405 return Ok(false);
2406 }
2407 row_cursor = row_cursor
2408 .checked_add(projection.capacity_size_bytes)
2409 .ok_or_else(|| invalid_resource("invocation wave row size overflows u64"))?;
2410 }
2411 peak = peak.max(row_cursor);
2412 if mode == InvocationLivenessMode::ConservativeConcurrent {
2413 concurrent_cursor = concurrent_cursor.checked_add(row_cursor).ok_or_else(|| {
2414 invalid_resource("concurrent invocation wave size overflows u64")
2415 })?;
2416 }
2417 }
2418 Ok(request.capacity_size_bytes
2419 == match mode {
2420 InvocationLivenessMode::TotalOrderReuse => peak,
2421 InvocationLivenessMode::ConservativeConcurrent => concurrent_cursor,
2422 InvocationLivenessMode::NoInvocationResources => 0,
2423 })
2424 }
2425
2426 pub(in crate::vnext::resource) fn view<'lease>(
2427 &'lease self,
2428 authority: &'lease LogicalBackingSliceAuthority,
2429 ) -> Result<LogicalBackingBufferView<'lease, R::Buffer>, VNextError> {
2430 self.view_many(std::slice::from_ref(authority))
2431 }
2432
2433 pub(in crate::vnext::resource) fn view_many<'lease>(
2434 &'lease self,
2435 authorities: &'lease [LogicalBackingSliceAuthority],
2436 ) -> Result<LogicalBackingBufferView<'lease, R::Buffer>, VNextError> {
2437 let first = authorities
2438 .first()
2439 .ok_or_else(|| invalid_resource("logical backing view requires an authority"))?;
2440 let pool = self
2441 .pools
2442 .get(&first.evidence.pool_id)
2443 .ok_or_else(|| invalid_resource("logical backing authority has no dynamic pool"))?;
2444 let mut logical_size_bytes = 0_u64;
2445 let mut capacity_size_bytes = 0_u64;
2446 let mut segment_count = 0_usize;
2447 for (index, authority) in authorities.iter().enumerate() {
2448 if authority.evidence.pool_id != first.evidence.pool_id
2449 || authority.evidence.resource_id != first.evidence.resource_id
2450 || authority.evidence.storage_profile != first.evidence.storage_profile
2451 || authority.evidence.alignment_bytes != first.evidence.alignment_bytes
2452 || authority.evidence.usage != first.evidence.usage
2453 || authority.evidence.element_type != first.evidence.element_type
2454 || authority.evidence.initialization != first.evidence.initialization
2455 {
2456 return Err(invalid_resource(
2457 "logical backing authorities have incompatible resource metadata",
2458 ));
2459 }
2460 Self::validate_authority(pool, authority)?;
2461 if index + 1 < authorities.len()
2462 && authority.evidence.logical_size_bytes != authority.evidence.capacity_size_bytes
2463 {
2464 return Err(invalid_resource(
2465 "multi-extent logical backing cannot contain interior capacity slack",
2466 ));
2467 }
2468 logical_size_bytes = logical_size_bytes
2469 .checked_add(authority.evidence.logical_size_bytes)
2470 .ok_or_else(|| invalid_resource("logical backing view size overflows u64"))?;
2471 capacity_size_bytes = capacity_size_bytes
2472 .checked_add(authority.evidence.capacity_size_bytes)
2473 .ok_or_else(|| invalid_resource("logical backing capacity overflows u64"))?;
2474 segment_count = segment_count
2475 .checked_add(authority.evidence.segments.len())
2476 .ok_or_else(|| invalid_resource("logical backing segment count overflows usize"))?;
2477 }
2478 let state = pool
2479 .state
2480 .lock()
2481 .map_err(|_| invalid_resource("dynamic backing pool is poisoned"))?;
2482 if state.poisoned {
2483 return Err(invalid_resource("dynamic backing pool is fail-closed"));
2484 }
2485 let mut bindings = Vec::with_capacity(segment_count);
2486 for authority in authorities {
2487 for segment in &authority.evidence.segments {
2488 let chunk = state.chunks.get(&segment.chunk_ordinal()).ok_or_else(|| {
2489 invalid_resource("logical backing references a missing chunk")
2490 })?;
2491 if segment.pool_id() != &authority.evidence.pool_id
2492 || chunk.backing.identity != *segment.chunk()
2493 || segment
2494 .offset_bytes()
2495 .checked_add(segment.length_bytes())
2496 .is_none_or(|end| end > chunk.backing.descriptor.size_bytes)
2497 {
2498 return Err(invalid_resource(
2499 "logical backing references a stale or out-of-bounds chunk region",
2500 ));
2501 }
2502 let retention = match authority.reusable_lane {
2503 Some(lane_id) => DeviceBufferRetention::lane_pair(
2504 lane_id,
2505 Arc::clone(&authority.segment_lease),
2506 Arc::clone(&chunk.backing),
2507 ),
2508 None => DeviceBufferRetention::pair(
2509 Arc::clone(&authority.segment_lease),
2510 Arc::clone(&chunk.backing),
2511 ),
2512 };
2513 bindings.push(LogicalBackingSegmentBinding {
2514 segment: segment.clone(),
2515 chunk: Arc::clone(&chunk.backing),
2516 retention,
2517 });
2518 }
2519 }
2520 drop(state);
2521 Ok(LogicalBackingBufferView {
2522 bindings,
2523 authorities,
2524 logical_size_bytes,
2525 capacity_size_bytes,
2526 alignment_bytes: first.evidence.alignment_bytes,
2527 usage: first.evidence.usage,
2528 element_type: first.evidence.element_type,
2529 storage_profile: first.evidence.storage_profile,
2530 })
2531 }
2532
2533 fn validate_authority(
2534 pool: &DynamicBackingPool<R>,
2535 authority: &LogicalBackingSliceAuthority,
2536 ) -> Result<(), VNextError> {
2537 if pool.instance_id != authority.evidence.pool_instance_id
2538 || authority.segment_lease.owner_instance_id != pool.instance_id
2539 || authority.segment_lease.owner.instance_id() != pool.instance_id
2540 || authority.segment_lease.claim_identity != authority.evidence.physical_claim_identity
2541 || authority.segment_lease.segment_generation != authority.evidence.segment_generation
2542 || authority.segment_lease.size_bytes != authority.evidence.physical_size_bytes
2543 || authority.evidence.domain_id != pool.domain.domain_id
2544 || authority.evidence.physical_claim_identity.pool_id() != pool.domain.pool_id()
2545 || authority
2546 .evidence
2547 .physical_claim_identity
2548 .resource_ids()
2549 .binary_search(&authority.evidence.resource_id)
2550 .is_err()
2551 || authority.evidence.logical_size_bytes == 0
2552 || authority.evidence.logical_size_bytes > authority.evidence.capacity_size_bytes
2553 || authority
2554 .evidence
2555 .physical_offset_bytes
2556 .checked_add(authority.evidence.capacity_size_bytes)
2557 .is_none_or(|end| end > authority.evidence.physical_size_bytes)
2558 || authority.evidence.storage_profile != pool.domain.pool.compatibility().profile()
2559 {
2560 return Err(invalid_resource(
2561 "logical backing authority belongs to another dynamic pool instance",
2562 ));
2563 }
2564 let expected_projection = backing_segment_range(
2565 &authority.segment_lease.segments,
2566 authority.evidence.physical_offset_bytes,
2567 authority.evidence.capacity_size_bytes,
2568 )?;
2569 if expected_projection != authority.evidence.segments {
2570 return Err(invalid_resource(
2571 "logical backing projection differs from its shared physical extent",
2572 ));
2573 }
2574 Ok(())
2575 }
2576}