1use serde::{Deserialize, Serialize};
2use std::collections::{BTreeMap, BTreeSet};
3use std::ops::{Deref, DerefMut};
4use std::sync::atomic::{AtomicU64, Ordering};
5use std::sync::{Arc, Mutex, MutexGuard};
6use tokio::sync::watch;
7
8use super::{CheckpointCapacityPolicy, DynamicAdmissionFaultKind, VNextError};
9
10mod checkpoint;
11use checkpoint::CheckpointClaimLedger;
12pub use checkpoint::{
13 CheckpointAuthorityId, CheckpointCapacityClaimDecision, CheckpointRetentionSkipReason,
14 LogicalCheckpointLease,
15};
16
17fn invalid_admission(reason: impl Into<String>) -> VNextError {
18 admission_fault(DynamicAdmissionFaultKind::InvalidContract, reason)
19}
20
21fn admission_fault(kind: DynamicAdmissionFaultKind, reason: impl Into<String>) -> VNextError {
22 VNextError::DynamicAdmissionContract {
23 kind,
24 reason: reason.into(),
25 }
26}
27
28static NEXT_COORDINATOR_ID: AtomicU64 = AtomicU64::new(1);
29
30#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
31#[serde(transparent)]
32pub struct LogicalAdmissionCoordinatorId(u64);
33
34impl LogicalAdmissionCoordinatorId {
35 fn issue() -> Result<Self, VNextError> {
36 NEXT_COORDINATOR_ID
37 .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
38 current.checked_add(1)
39 })
40 .map(Self)
41 .map_err(|_| {
42 admission_fault(
43 DynamicAdmissionFaultKind::AuthorityExhausted,
44 "logical admission coordinator id is exhausted",
45 )
46 })
47 }
48
49 pub const fn get(self) -> u64 {
50 self.0
51 }
52}
53
54#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
55#[serde(transparent)]
56pub struct CapacityDomainId(u32);
57
58impl CapacityDomainId {
59 pub fn new(value: u32) -> Result<Self, VNextError> {
60 if value == 0 {
61 return Err(invalid_admission("capacity domain id must be non-zero"));
62 }
63 Ok(Self(value))
64 }
65
66 pub const fn get(self) -> u32 {
67 self.0
68 }
69}
70
71#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)]
72#[serde(transparent)]
73pub struct CapacityUnits(u64);
74
75impl CapacityUnits {
76 pub const ZERO: Self = Self(0);
77
78 pub const fn new(value: u64) -> Self {
79 Self(value)
80 }
81
82 pub const fn get(self) -> u64 {
83 self.0
84 }
85}
86
87#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
88pub struct CapacityDomainSpec {
89 total_units: CapacityUnits,
90 maximum_total_units: CapacityUnits,
91}
92
93impl CapacityDomainSpec {
94 pub fn new(
95 total_units: CapacityUnits,
96 maximum_total_units: CapacityUnits,
97 ) -> Result<Self, VNextError> {
98 if total_units.get() > maximum_total_units.get() {
99 return Err(invalid_admission(
100 "capacity domain total exceeds maximum total",
101 ));
102 }
103 Ok(Self {
104 total_units,
105 maximum_total_units,
106 })
107 }
108
109 pub const fn total_units(self) -> CapacityUnits {
110 self.total_units
111 }
112
113 pub const fn maximum_total_units(self) -> CapacityUnits {
114 self.maximum_total_units
115 }
116}
117
118#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
119pub struct CapacityEntry {
120 domain: CapacityDomainId,
121 units: CapacityUnits,
122}
123
124impl CapacityEntry {
125 pub fn new(domain: CapacityDomainId, units: CapacityUnits) -> Result<Self, VNextError> {
126 if units.get() == 0 {
127 return Err(invalid_admission("capacity demand units must be non-zero"));
128 }
129 Ok(Self { domain, units })
130 }
131
132 pub const fn domain(self) -> CapacityDomainId {
133 self.domain
134 }
135
136 pub const fn units(self) -> CapacityUnits {
137 self.units
138 }
139}
140
141#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
142#[serde(transparent)]
143pub struct CapacityVector(Vec<CapacityEntry>);
144
145impl CapacityVector {
146 pub fn new(mut entries: Vec<CapacityEntry>) -> Result<Self, VNextError> {
147 entries.sort_by_key(|entry| entry.domain);
148 if entries.is_empty() {
149 return Err(invalid_admission("capacity vector must be non-empty"));
150 }
151 if entries
152 .windows(2)
153 .any(|pair| pair[0].domain == pair[1].domain)
154 {
155 return Err(invalid_admission(
156 "capacity vector contains a duplicate domain",
157 ));
158 }
159 Ok(Self(entries))
160 }
161
162 pub fn entries(&self) -> &[CapacityEntry] {
163 &self.0
164 }
165
166 pub(crate) const fn empty() -> Self {
167 Self(Vec::new())
168 }
169
170 pub const fn is_empty(&self) -> bool {
171 self.0.is_empty()
172 }
173
174 pub(crate) fn units_for(&self, domain: CapacityDomainId) -> Option<CapacityUnits> {
175 self.0
176 .binary_search_by_key(&domain, |entry| entry.domain)
177 .ok()
178 .map(|index| self.0[index].units)
179 }
180
181 pub(crate) fn checked_sum(left: &Self, right: &Self) -> Result<Self, VNextError> {
182 let mut units_by_domain = BTreeMap::<CapacityDomainId, u64>::new();
183 for entry in left.entries().iter().chain(right.entries()) {
184 let units = units_by_domain.entry(entry.domain()).or_default();
185 *units = units.checked_add(entry.units().get()).ok_or_else(|| {
186 admission_fault(
187 DynamicAdmissionFaultKind::ArithmeticOverflow,
188 "combined admission demand overflows u64",
189 )
190 })?;
191 }
192 if units_by_domain.is_empty() {
193 return Ok(Self::empty());
194 }
195 Self::new(
196 units_by_domain
197 .into_iter()
198 .map(|(domain, units)| CapacityEntry::new(domain, CapacityUnits::new(units)))
199 .collect::<Result<Vec<_>, _>>()?,
200 )
201 }
202}
203
204#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
205#[serde(rename_all = "snake_case")]
206pub enum AdmissionFitPolicy {
207 ImmediateOnly,
208 FullInputMustFit,
211}
212
213#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
214#[serde(rename_all = "snake_case")]
215pub enum AdmissionPressureAction {
216 WaitForRelease,
217 PreemptAndRecompute,
218}
219
220#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
223pub struct AdmissionDemand {
224 immediate_claim: CapacityVector,
225 fit_requirement: CapacityVector,
226 fit_policy: AdmissionFitPolicy,
227 pressure_action: AdmissionPressureAction,
228}
229
230impl AdmissionDemand {
231 pub(crate) fn from_plan(
232 immediate_claim: CapacityVector,
233 fit_requirement: CapacityVector,
234 fit_policy: AdmissionFitPolicy,
235 pressure_action: AdmissionPressureAction,
236 ) -> Result<Self, VNextError> {
237 for immediate in immediate_claim.entries() {
238 let Some(fit_units) = fit_requirement.units_for(immediate.domain) else {
239 return Err(invalid_admission(
240 "fit requirement omits an immediate-claim domain",
241 ));
242 };
243 if fit_units.get() < immediate.units.get() {
244 return Err(invalid_admission(
245 "fit requirement is smaller than the immediate claim",
246 ));
247 }
248 }
249 if fit_policy == AdmissionFitPolicy::ImmediateOnly && fit_requirement != immediate_claim {
250 return Err(invalid_admission(
251 "immediate-only admission requires identical immediate and fit vectors",
252 ));
253 }
254 Ok(Self {
255 immediate_claim,
256 fit_requirement,
257 fit_policy,
258 pressure_action,
259 })
260 }
261
262 pub fn immediate_claim(&self) -> &CapacityVector {
263 &self.immediate_claim
264 }
265
266 pub fn fit_requirement(&self) -> &CapacityVector {
267 &self.fit_requirement
268 }
269
270 pub const fn fit_policy(&self) -> AdmissionFitPolicy {
271 self.fit_policy
272 }
273
274 pub const fn pressure_action(&self) -> AdmissionPressureAction {
275 self.pressure_action
276 }
277
278 fn initial_sequence_bundle(request: &Self, sequence: &Self) -> Result<Self, VNextError> {
279 if request.pressure_action != sequence.pressure_action {
280 return Err(invalid_admission(
281 "initial request and sequence require one pressure action",
282 ));
283 }
284 let fit_policy = if request.fit_policy == AdmissionFitPolicy::FullInputMustFit
285 || sequence.fit_policy == AdmissionFitPolicy::FullInputMustFit
286 {
287 AdmissionFitPolicy::FullInputMustFit
288 } else {
289 AdmissionFitPolicy::ImmediateOnly
290 };
291 Self::from_plan(
292 CapacityVector::checked_sum(&request.immediate_claim, &sequence.immediate_claim)?,
293 CapacityVector::checked_sum(&request.fit_requirement, &sequence.fit_requirement)?,
294 fit_policy,
295 request.pressure_action,
296 )
297 }
298}
299
300#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
301#[serde(rename_all = "snake_case")]
302pub enum CapacityShortfallKind {
303 ImmediateAvailability,
304 FitAvailability,
305 BackingGrowthRequired,
306 ActiveSequenceCeiling,
307 PermanentDomainMaximum,
308 PermanentPlanBudget,
309}
310
311#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
312pub struct CapacityShortfall {
313 domain: Option<CapacityDomainId>,
314 kind: CapacityShortfallKind,
315 requested: CapacityUnits,
316 available: CapacityUnits,
317 current_total: CapacityUnits,
318 maximum_total: CapacityUnits,
319}
320
321impl CapacityShortfall {
322 pub const fn domain(&self) -> Option<CapacityDomainId> {
323 self.domain
324 }
325
326 pub const fn kind(&self) -> CapacityShortfallKind {
327 self.kind
328 }
329
330 pub const fn requested(&self) -> CapacityUnits {
331 self.requested
332 }
333
334 pub const fn available(&self) -> CapacityUnits {
335 self.available
336 }
337
338 pub const fn current_total(&self) -> CapacityUnits {
339 self.current_total
340 }
341
342 pub const fn maximum_total(&self) -> CapacityUnits {
343 self.maximum_total
344 }
345}
346
347#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
348pub struct DomainCapacitySnapshot {
349 domain: CapacityDomainId,
350 total: CapacityUnits,
351 maximum_total: CapacityUnits,
352 used: CapacityUnits,
353 available: CapacityUnits,
354}
355
356impl DomainCapacitySnapshot {
357 pub const fn domain(&self) -> CapacityDomainId {
358 self.domain
359 }
360
361 pub const fn total(&self) -> CapacityUnits {
362 self.total
363 }
364
365 pub const fn maximum_total(&self) -> CapacityUnits {
366 self.maximum_total
367 }
368
369 pub const fn used(&self) -> CapacityUnits {
370 self.used
371 }
372
373 pub const fn available(&self) -> CapacityUnits {
374 self.available
375 }
376}
377
378#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
379pub struct CapacitySnapshot {
380 coordinator_id: LogicalAdmissionCoordinatorId,
381 domains: Vec<DomainCapacitySnapshot>,
382 active_requests: u32,
383 active_sequences: u32,
384 active_child_claims: u64,
385 active_checkpoint_claims: u64,
386 maximum_active_sequences: u32,
387 release_epoch: u64,
388 capacity_epoch: u64,
389 live_sequence_records: usize,
390 reusable_sequence_ids: usize,
391 live_request_records: usize,
392 reusable_request_ids: usize,
393 poisoned: bool,
394}
395
396impl CapacitySnapshot {
397 pub const fn coordinator_id(&self) -> LogicalAdmissionCoordinatorId {
398 self.coordinator_id
399 }
400
401 pub fn domains(&self) -> &[DomainCapacitySnapshot] {
402 &self.domains
403 }
404
405 pub const fn active_sequences(&self) -> u32 {
406 self.active_sequences
407 }
408
409 pub const fn active_requests(&self) -> u32 {
410 self.active_requests
411 }
412
413 pub const fn active_child_claims(&self) -> u64 {
414 self.active_child_claims
415 }
416
417 pub const fn active_checkpoint_claims(&self) -> u64 {
420 self.active_checkpoint_claims
421 }
422
423 pub const fn maximum_active_sequences(&self) -> u32 {
424 self.maximum_active_sequences
425 }
426
427 pub const fn release_epoch(&self) -> u64 {
428 self.release_epoch
429 }
430
431 pub const fn capacity_epoch(&self) -> u64 {
432 self.capacity_epoch
433 }
434
435 pub const fn live_sequence_records(&self) -> usize {
436 self.live_sequence_records
437 }
438
439 pub const fn reusable_sequence_ids(&self) -> usize {
440 self.reusable_sequence_ids
441 }
442
443 pub const fn live_request_records(&self) -> usize {
444 self.live_request_records
445 }
446
447 pub const fn reusable_request_ids(&self) -> usize {
448 self.reusable_request_ids
449 }
450
451 pub const fn poisoned(&self) -> bool {
452 self.poisoned
453 }
454}
455
456#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
457pub struct CapacityEpochs {
458 coordinator_id: LogicalAdmissionCoordinatorId,
459 release_epoch: u64,
460 capacity_epoch: u64,
461}
462
463impl CapacityEpochs {
464 pub const fn coordinator_id(self) -> LogicalAdmissionCoordinatorId {
465 self.coordinator_id
466 }
467
468 pub const fn release_epoch(self) -> u64 {
469 self.release_epoch
470 }
471
472 pub const fn capacity_epoch(self) -> u64 {
473 self.capacity_epoch
474 }
475}
476
477#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
481#[serde(rename_all = "snake_case")]
482pub enum CapacityAvailabilitySource {
483 Domain(CapacityDomainId),
484 ActiveSequenceSlots,
485 PlanDeviceBudget,
486 ProcessDeviceCapacity,
487}
488
489#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)]
490pub struct CapacityAvailabilityEpoch {
491 source: CapacityAvailabilitySource,
492 epoch: u64,
493}
494
495impl CapacityAvailabilityEpoch {
496 pub fn new(source: CapacityAvailabilitySource, epoch: u64) -> Result<Self, VNextError> {
497 if epoch == 0 {
498 return Err(invalid_admission(
499 "capacity availability epoch must be non-zero",
500 ));
501 }
502 Ok(Self { source, epoch })
503 }
504
505 pub const fn source(self) -> CapacityAvailabilitySource {
506 self.source
507 }
508
509 pub const fn epoch(self) -> u64 {
510 self.epoch
511 }
512}
513
514#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
518pub struct CapacityWaitCondition {
519 coordinator_id: LogicalAdmissionCoordinatorId,
520 observed: Vec<CapacityAvailabilityEpoch>,
521}
522
523impl CapacityWaitCondition {
524 pub fn from_observation(
525 coordinator_id: u64,
526 observed: Vec<CapacityAvailabilityEpoch>,
527 ) -> Result<Self, VNextError> {
528 if coordinator_id == 0 {
529 return Err(invalid_admission(
530 "capacity wait coordinator id must be non-zero",
531 ));
532 }
533 Self::new(LogicalAdmissionCoordinatorId(coordinator_id), observed)
534 }
535
536 pub fn new(
537 coordinator_id: LogicalAdmissionCoordinatorId,
538 mut observed: Vec<CapacityAvailabilityEpoch>,
539 ) -> Result<Self, VNextError> {
540 if observed.is_empty() {
541 return Err(invalid_admission(
542 "capacity wait condition requires at least one availability source",
543 ));
544 }
545 observed.sort_by_key(|entry| entry.source);
546 if observed
547 .windows(2)
548 .any(|pair| pair[0].source == pair[1].source)
549 {
550 return Err(invalid_admission(
551 "capacity wait condition contains a duplicate availability source",
552 ));
553 }
554 Ok(Self {
555 coordinator_id,
556 observed,
557 })
558 }
559
560 pub const fn coordinator_id(&self) -> LogicalAdmissionCoordinatorId {
561 self.coordinator_id
562 }
563
564 pub fn observed(&self) -> &[CapacityAvailabilityEpoch] {
565 &self.observed
566 }
567
568 pub fn validate_sources_present(
569 &self,
570 current: &[CapacityAvailabilityEpoch],
571 ) -> Result<(), VNextError> {
572 if current
573 .windows(2)
574 .any(|pair| pair[0].source >= pair[1].source)
575 {
576 return Err(invalid_admission(
577 "capacity availability snapshot is not canonical",
578 ));
579 }
580 for observed in &self.observed {
581 current
582 .binary_search_by_key(&observed.source, |entry| entry.source)
583 .map_err(|_| {
584 invalid_admission("capacity availability snapshot omitted a waited source")
585 })?;
586 }
587 Ok(())
588 }
589
590 pub fn changed_since(&self, current: &[CapacityAvailabilityEpoch]) -> Result<bool, VNextError> {
591 self.validate_sources_present(current)?;
592 let mut changed = false;
593 for observed in &self.observed {
594 let index = current
595 .binary_search_by_key(&observed.source, |entry| entry.source)
596 .expect("validated availability source remains present");
597 let current_epoch = current[index].epoch;
598 if current_epoch < observed.epoch {
599 return Err(admission_fault(
600 DynamicAdmissionFaultKind::EpochRegression,
601 "capacity availability epoch regressed",
602 ));
603 }
604 changed |= current_epoch > observed.epoch;
605 }
606 Ok(changed)
607 }
608
609 pub(crate) fn refreshed_from(
610 &self,
611 current: &[CapacityAvailabilityEpoch],
612 ) -> Result<Self, VNextError> {
613 self.validate_sources_present(current)?;
614 Self::new(
615 self.coordinator_id,
616 self.observed
617 .iter()
618 .map(|observed| {
619 let index = current
620 .binary_search_by_key(&observed.source, |entry| entry.source)
621 .expect("validated availability source remains present");
622 current[index]
623 })
624 .collect(),
625 )
626 }
627}
628
629#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
636pub struct CapacityWaitSnapshot {
637 epochs: CapacityEpochs,
638 wait_condition: CapacityWaitCondition,
639}
640
641impl CapacityWaitSnapshot {
642 fn new(epochs: CapacityEpochs, wait_condition: CapacityWaitCondition) -> Self {
643 debug_assert_eq!(epochs.coordinator_id(), wait_condition.coordinator_id());
644 Self {
645 epochs,
646 wait_condition,
647 }
648 }
649
650 pub const fn epochs(&self) -> CapacityEpochs {
651 self.epochs
652 }
653
654 pub fn wait_condition(&self) -> &CapacityWaitCondition {
655 &self.wait_condition
656 }
657
658 pub(crate) fn narrow_to_domains(
659 self,
660 domains: impl IntoIterator<Item = CapacityDomainId>,
661 ) -> Result<Self, VNextError> {
662 let sources = domains
663 .into_iter()
664 .map(CapacityAvailabilitySource::Domain)
665 .collect::<BTreeSet<_>>();
666 if sources.is_empty() {
667 return Err(invalid_admission(
668 "capacity wait snapshot cannot be narrowed to no domains",
669 ));
670 }
671 let observed = sources
672 .into_iter()
673 .map(|source| {
674 self.wait_condition
675 .observed
676 .binary_search_by_key(&source, |entry| entry.source)
677 .map(|index| self.wait_condition.observed[index])
678 .map_err(|_| {
679 invalid_admission("capacity wait snapshot cannot add an unobserved domain")
680 })
681 })
682 .collect::<Result<Vec<_>, _>>()?;
683 Ok(Self::new(
684 self.epochs,
685 CapacityWaitCondition::new(self.wait_condition.coordinator_id, observed)?,
686 ))
687 }
688}
689
690#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
691pub struct SequenceAuthorityId {
692 sparse_id: u32,
693 generation: u64,
694}
695
696#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
697pub struct RequestAuthorityId {
698 sparse_id: u32,
699 generation: u64,
700}
701
702impl RequestAuthorityId {
703 #[cfg(test)]
704 pub(crate) const fn test_only(sparse_id: u32, generation: u64) -> Self {
705 Self {
706 sparse_id,
707 generation,
708 }
709 }
710
711 pub const fn sparse_id(self) -> u32 {
712 self.sparse_id
713 }
714
715 pub const fn generation(self) -> u64 {
716 self.generation
717 }
718}
719
720impl SequenceAuthorityId {
721 #[cfg(test)]
722 pub(crate) const fn test_only(sparse_id: u32, generation: u64) -> Self {
723 Self {
724 sparse_id,
725 generation,
726 }
727 }
728
729 pub const fn sparse_id(self) -> u32 {
730 self.sparse_id
731 }
732
733 pub const fn generation(self) -> u64 {
734 self.generation
735 }
736}
737
738#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
739#[serde(rename_all = "snake_case")]
740pub enum DeferredAction {
741 WaitForRelease,
742 AwaitBackingGrowth,
743 PreemptAndRecompute,
744}
745
746#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
747pub struct AdmissionDeferred {
748 immediate_requested: CapacityVector,
749 fit_requested: CapacityVector,
750 available: CapacitySnapshot,
751 blockers: Vec<CapacityShortfall>,
752 action: DeferredAction,
753 release_epoch: u64,
754 capacity_epoch: u64,
755 wait_condition: CapacityWaitCondition,
756}
757
758impl AdmissionDeferred {
759 pub fn immediate_requested(&self) -> &CapacityVector {
760 &self.immediate_requested
761 }
762
763 pub fn fit_requested(&self) -> &CapacityVector {
764 &self.fit_requested
765 }
766
767 pub fn available(&self) -> &CapacitySnapshot {
768 &self.available
769 }
770
771 pub fn blockers(&self) -> &[CapacityShortfall] {
772 &self.blockers
773 }
774
775 pub const fn action(&self) -> DeferredAction {
776 self.action
777 }
778
779 pub const fn release_epoch(&self) -> u64 {
780 self.release_epoch
781 }
782
783 pub const fn capacity_epoch(&self) -> u64 {
784 self.capacity_epoch
785 }
786
787 pub const fn epochs(&self) -> CapacityEpochs {
788 CapacityEpochs {
789 coordinator_id: self.available.coordinator_id,
790 release_epoch: self.release_epoch,
791 capacity_epoch: self.capacity_epoch,
792 }
793 }
794
795 pub fn wait_condition(&self) -> &CapacityWaitCondition {
796 &self.wait_condition
797 }
798}
799
800#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
801pub struct AdmissionRejected {
802 immediate_requested: CapacityVector,
803 fit_requested: CapacityVector,
804 maximum: CapacitySnapshot,
805 blockers: Vec<CapacityShortfall>,
806}
807
808impl AdmissionRejected {
809 pub(crate) fn for_plan_budget(
810 immediate_requested: CapacityVector,
811 fit_requested: CapacityVector,
812 maximum: CapacitySnapshot,
813 minimum_required_bytes: u64,
814 usable_bytes: u64,
815 ) -> Self {
816 debug_assert!(minimum_required_bytes > usable_bytes);
817 Self {
818 immediate_requested,
819 fit_requested,
820 maximum,
821 blockers: vec![CapacityShortfall {
822 domain: None,
823 kind: CapacityShortfallKind::PermanentPlanBudget,
824 requested: CapacityUnits::new(minimum_required_bytes),
825 available: CapacityUnits::new(usable_bytes),
826 current_total: CapacityUnits::new(usable_bytes),
827 maximum_total: CapacityUnits::new(usable_bytes),
828 }],
829 }
830 }
831
832 pub fn blockers(&self) -> &[CapacityShortfall] {
833 &self.blockers
834 }
835
836 pub fn maximum(&self) -> &CapacitySnapshot {
837 &self.maximum
838 }
839}
840
841#[derive(Debug)]
842pub enum AdmissionDecision {
843 Admitted(LogicalAdmissionLease),
844 Deferred(AdmissionDeferred),
845 PermanentRejected(AdmissionRejected),
846}
847
848#[derive(Debug)]
849pub enum RequestAdmissionDecision {
850 Admitted(LogicalRequestLease),
851 Deferred(AdmissionDeferred),
852 PermanentRejected(AdmissionRejected),
853}
854
855pub(crate) enum InitialSequenceAdmissionDecision {
856 Admitted(LogicalInitialSequenceAdmission),
857 Deferred,
858 PermanentRejected(AdmissionRejected),
859}
860
861pub(crate) struct LogicalInitialSequenceAdmission {
865 sequence: LogicalAdmissionLease,
866 request: LogicalRequestLease,
867}
868
869impl LogicalInitialSequenceAdmission {
870 pub(crate) fn into_parts(self) -> (LogicalRequestLease, LogicalAdmissionLease) {
871 (self.request, self.sequence)
872 }
873}
874
875pub(crate) enum AdmissionPreflightDecision {
876 Eligible,
877 Deferred(AdmissionDeferred),
878 PermanentRejected(AdmissionRejected),
879}
880
881#[derive(Debug)]
882pub enum CapacityClaimDecision {
883 Claimed(LogicalCapacityLease),
884 Deferred(AdmissionDeferred),
885 PermanentRejected(AdmissionRejected),
886}
887
888#[derive(Debug)]
889pub enum BatchCapacityClaimDecision {
890 Claimed(LogicalBatchCapacityLease),
891 Deferred(AdmissionDeferred),
892 PermanentRejected(AdmissionRejected),
893}
894
895#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
900pub struct SequenceCapacityParent {
901 request: RequestAuthorityId,
902 sequence: SequenceAuthorityId,
903}
904
905impl SequenceCapacityParent {
906 pub const fn request(self) -> RequestAuthorityId {
907 self.request
908 }
909
910 pub const fn sequence(self) -> SequenceAuthorityId {
911 self.sequence
912 }
913}
914
915#[derive(Debug)]
916struct DomainState {
917 spec: CapacityDomainSpec,
918 used: u64,
919 availability_epoch: u64,
920}
921
922#[derive(Debug, Clone, Copy)]
923struct LiveSequenceRecord {
924 generation: u64,
925 request: RequestAuthorityId,
926 active_child_claims: u64,
927}
928
929#[derive(Debug, Clone, Copy)]
930struct LiveRequestRecord {
931 generation: u64,
932 active_sequences: u32,
933}
934
935#[derive(Debug)]
936struct CoordinatorState {
937 domains: BTreeMap<CapacityDomainId, DomainState>,
938 maximum_active_sequences: u32,
939 active_requests: u32,
940 active_sequences: u32,
941 active_child_claims: u64,
942 checkpoint_claims: CheckpointClaimLedger,
943 live_requests: Vec<Option<LiveRequestRecord>>,
944 reusable_request_ids: Vec<u32>,
945 live_sequences: Vec<Option<LiveSequenceRecord>>,
946 reusable_sequence_ids: Vec<u32>,
947 next_request_generation: u64,
948 next_sequence_generation: u64,
949 release_epoch: u64,
950 capacity_epoch: u64,
951 active_sequence_availability_epoch: u64,
952 poisoned: bool,
953}
954
955impl CoordinatorState {
956 fn snapshot(&self, coordinator_id: LogicalAdmissionCoordinatorId) -> CapacitySnapshot {
957 let domains = self
958 .domains
959 .iter()
960 .map(|(domain, state)| {
961 let total = state.spec.total_units.get();
962 DomainCapacitySnapshot {
963 domain: *domain,
964 total: CapacityUnits::new(total),
965 maximum_total: state.spec.maximum_total_units,
966 used: CapacityUnits::new(state.used),
967 available: CapacityUnits::new(total.saturating_sub(state.used)),
968 }
969 })
970 .collect();
971 CapacitySnapshot {
972 coordinator_id,
973 domains,
974 active_requests: self.active_requests,
975 active_sequences: self.active_sequences,
976 active_child_claims: self.active_child_claims,
977 active_checkpoint_claims: self.checkpoint_claims.count(),
978 maximum_active_sequences: self.maximum_active_sequences,
979 release_epoch: self.release_epoch,
980 capacity_epoch: self.capacity_epoch,
981 live_sequence_records: if self.poisoned {
982 self.live_sequences
983 .iter()
984 .filter(|record| record.is_some())
985 .count()
986 } else {
987 self.active_sequences as usize
988 },
989 reusable_sequence_ids: self.reusable_sequence_ids.len(),
990 live_request_records: if self.poisoned {
991 self.live_requests
992 .iter()
993 .filter(|record| record.is_some())
994 .count()
995 } else {
996 self.active_requests as usize
997 },
998 reusable_request_ids: self.reusable_request_ids.len(),
999 poisoned: self.poisoned,
1000 }
1001 }
1002
1003 fn epochs(&self, coordinator_id: LogicalAdmissionCoordinatorId) -> CapacityEpochs {
1004 CapacityEpochs {
1005 coordinator_id,
1006 release_epoch: self.release_epoch,
1007 capacity_epoch: self.capacity_epoch,
1008 }
1009 }
1010
1011 fn write_availability_epochs(&self, out: &mut Vec<CapacityAvailabilityEpoch>) {
1012 out.clear();
1013 out.extend(
1014 self.domains
1015 .iter()
1016 .map(|(domain, state)| CapacityAvailabilityEpoch {
1017 source: CapacityAvailabilitySource::Domain(*domain),
1018 epoch: state.availability_epoch,
1019 }),
1020 );
1021 out.push(CapacityAvailabilityEpoch {
1022 source: CapacityAvailabilitySource::ActiveSequenceSlots,
1023 epoch: self.active_sequence_availability_epoch,
1024 });
1025 }
1026
1027 fn wait_condition_for_blockers(
1028 &self,
1029 coordinator_id: LogicalAdmissionCoordinatorId,
1030 blockers: &[CapacityShortfall],
1031 ) -> Result<CapacityWaitCondition, VNextError> {
1032 let mut sources = BTreeSet::new();
1033 for blocker in blockers {
1034 match (blocker.kind, blocker.domain) {
1035 (CapacityShortfallKind::ActiveSequenceCeiling, None) => {
1036 sources.insert(CapacityAvailabilitySource::ActiveSequenceSlots);
1037 }
1038 (
1039 CapacityShortfallKind::PermanentDomainMaximum
1040 | CapacityShortfallKind::PermanentPlanBudget,
1041 _,
1042 ) => {
1043 return Err(invalid_admission(
1044 "permanent capacity blocker cannot produce a wait condition",
1045 ));
1046 }
1047 (_, Some(domain)) => {
1048 if !self.domains.contains_key(&domain) {
1049 return Err(admission_fault(
1050 DynamicAdmissionFaultKind::UnknownDomain,
1051 "capacity blocker references an unknown wait domain",
1052 ));
1053 }
1054 sources.insert(CapacityAvailabilitySource::Domain(domain));
1055 }
1056 (_, None) => {
1057 return Err(invalid_admission(
1058 "capacity blocker contains no availability source",
1059 ));
1060 }
1061 }
1062 }
1063 self.wait_condition_for_sources(coordinator_id, sources)
1064 }
1065
1066 fn wait_condition_for_domains(
1067 &self,
1068 coordinator_id: LogicalAdmissionCoordinatorId,
1069 domains: impl IntoIterator<Item = CapacityDomainId>,
1070 ) -> Result<CapacityWaitCondition, VNextError> {
1071 self.wait_condition_for_sources(
1072 coordinator_id,
1073 domains
1074 .into_iter()
1075 .map(CapacityAvailabilitySource::Domain)
1076 .collect(),
1077 )
1078 }
1079
1080 fn wait_condition_for_sources(
1081 &self,
1082 coordinator_id: LogicalAdmissionCoordinatorId,
1083 sources: BTreeSet<CapacityAvailabilitySource>,
1084 ) -> Result<CapacityWaitCondition, VNextError> {
1085 let observed = sources
1086 .into_iter()
1087 .map(|source| {
1088 let epoch = match source {
1089 CapacityAvailabilitySource::Domain(domain) => {
1090 self.domains
1091 .get(&domain)
1092 .ok_or_else(|| {
1093 admission_fault(
1094 DynamicAdmissionFaultKind::UnknownDomain,
1095 "capacity wait references an unknown domain",
1096 )
1097 })?
1098 .availability_epoch
1099 }
1100 CapacityAvailabilitySource::ActiveSequenceSlots => {
1101 self.active_sequence_availability_epoch
1102 }
1103 CapacityAvailabilitySource::PlanDeviceBudget
1104 | CapacityAvailabilitySource::ProcessDeviceCapacity => {
1105 return Err(invalid_admission(
1106 "device capacity availability is owned by the device account",
1107 ));
1108 }
1109 };
1110 CapacityAvailabilityEpoch::new(source, epoch)
1111 })
1112 .collect::<Result<Vec<_>, _>>()?;
1113 CapacityWaitCondition::new(coordinator_id, observed)
1114 }
1115
1116 fn availability_epoch_for(
1117 &self,
1118 source: CapacityAvailabilitySource,
1119 ) -> Result<u64, VNextError> {
1120 match source {
1121 CapacityAvailabilitySource::Domain(domain) => self
1122 .domains
1123 .get(&domain)
1124 .map(|state| state.availability_epoch)
1125 .ok_or_else(|| {
1126 admission_fault(
1127 DynamicAdmissionFaultKind::UnknownDomain,
1128 "capacity wait references an unknown domain",
1129 )
1130 }),
1131 CapacityAvailabilitySource::ActiveSequenceSlots => {
1132 Ok(self.active_sequence_availability_epoch)
1133 }
1134 CapacityAvailabilitySource::PlanDeviceBudget
1135 | CapacityAvailabilitySource::ProcessDeviceCapacity => Err(invalid_admission(
1136 "device capacity availability is owned by the device account",
1137 )),
1138 }
1139 }
1140
1141 fn wait_condition_changed(
1142 &self,
1143 coordinator_id: LogicalAdmissionCoordinatorId,
1144 observed: &CapacityWaitCondition,
1145 ) -> Result<bool, VNextError> {
1146 if observed.coordinator_id != coordinator_id {
1147 return Err(admission_fault(
1148 DynamicAdmissionFaultKind::ForeignCoordinator,
1149 "capacity wait condition belongs to a different coordinator",
1150 ));
1151 }
1152 let mut changed = false;
1153 for entry in &observed.observed {
1154 let current = self.availability_epoch_for(entry.source)?;
1155 if current < entry.epoch {
1156 return Err(admission_fault(
1157 DynamicAdmissionFaultKind::EpochRegression,
1158 "capacity availability epoch regressed",
1159 ));
1160 }
1161 changed |= current > entry.epoch;
1162 }
1163 Ok(changed)
1164 }
1165
1166 fn refresh_wait_condition(
1167 &self,
1168 coordinator_id: LogicalAdmissionCoordinatorId,
1169 observed: &CapacityWaitCondition,
1170 ) -> Result<CapacityWaitCondition, VNextError> {
1171 if observed.coordinator_id != coordinator_id {
1172 return Err(admission_fault(
1173 DynamicAdmissionFaultKind::ForeignCoordinator,
1174 "capacity wait condition belongs to a different coordinator",
1175 ));
1176 }
1177 CapacityWaitCondition::new(
1178 coordinator_id,
1179 observed
1180 .observed
1181 .iter()
1182 .map(|entry| {
1183 CapacityAvailabilityEpoch::new(
1184 entry.source,
1185 self.availability_epoch_for(entry.source)?,
1186 )
1187 })
1188 .collect::<Result<Vec<_>, _>>()?,
1189 )
1190 }
1191
1192 fn preview_request_authority(&self) -> Result<RequestAuthorityReservation, VNextError> {
1193 let generation = self.next_request_generation;
1194 let next_generation = generation.checked_add(1).ok_or_else(|| {
1195 admission_fault(
1196 DynamicAdmissionFaultKind::AuthorityExhausted,
1197 "request authority generation is exhausted",
1198 )
1199 })?;
1200 let (sparse_id, source) = if let Some(id) = self.reusable_request_ids.last().copied() {
1201 (id, SparseIdSource::Reused)
1202 } else {
1203 if self.live_requests.len() >= u32::MAX as usize {
1204 return Err(admission_fault(
1205 DynamicAdmissionFaultKind::AuthorityExhausted,
1206 "request authority id space is exhausted",
1207 ));
1208 }
1209 (self.live_requests.len() as u32, SparseIdSource::Fresh)
1210 };
1211 if self
1212 .live_requests
1213 .get(sparse_id as usize)
1214 .is_some_and(Option::is_some)
1215 {
1216 return Err(invalid_admission(
1217 "request authority allocator selected a live sparse id",
1218 ));
1219 }
1220 Ok(RequestAuthorityReservation {
1221 authority: RequestAuthorityId {
1222 sparse_id,
1223 generation,
1224 },
1225 source,
1226 next_generation,
1227 })
1228 }
1229
1230 fn prepare_request_storage(
1231 &mut self,
1232 reservation: RequestAuthorityReservation,
1233 ) -> Result<(), VNextError> {
1234 if matches!(reservation.source, SparseIdSource::Fresh) {
1235 self.live_requests.try_reserve(1).map_err(|_| {
1236 admission_fault(
1237 DynamicAdmissionFaultKind::AllocationFailure,
1238 "cannot reserve request authority slab",
1239 )
1240 })?;
1241 self.reusable_request_ids
1242 .try_reserve(self.live_requests.len() + 1)
1243 .map_err(|_| {
1244 admission_fault(
1245 DynamicAdmissionFaultKind::AllocationFailure,
1246 "cannot reserve request release free-list",
1247 )
1248 })?;
1249 }
1250 Ok(())
1251 }
1252
1253 fn commit_request_authority(
1254 &mut self,
1255 reservation: RequestAuthorityReservation,
1256 ) -> RequestAuthorityId {
1257 let record = LiveRequestRecord {
1258 generation: reservation.authority.generation,
1259 active_sequences: 0,
1260 };
1261 match reservation.source {
1262 SparseIdSource::Reused => {
1263 self.reusable_request_ids.pop();
1264 self.live_requests[reservation.authority.sparse_id as usize] = Some(record);
1265 }
1266 SparseIdSource::Fresh => self.live_requests.push(Some(record)),
1267 }
1268 self.next_request_generation = reservation.next_generation;
1269 reservation.authority
1270 }
1271
1272 fn preview_sequence_authority(&self) -> Result<SequenceAuthorityReservation, VNextError> {
1273 let generation = self.next_sequence_generation;
1274 let next_generation = generation.checked_add(1).ok_or_else(|| {
1275 admission_fault(
1276 DynamicAdmissionFaultKind::AuthorityExhausted,
1277 "sequence authority generation is exhausted",
1278 )
1279 })?;
1280 let (sparse_id, source) = if let Some(id) = self.reusable_sequence_ids.last().copied() {
1281 (id, SparseIdSource::Reused)
1282 } else {
1283 if self.live_sequences.len() >= u32::MAX as usize {
1284 return Err(admission_fault(
1285 DynamicAdmissionFaultKind::AuthorityExhausted,
1286 "sequence authority id space is exhausted",
1287 ));
1288 }
1289 (self.live_sequences.len() as u32, SparseIdSource::Fresh)
1290 };
1291 if self
1292 .live_sequences
1293 .get(sparse_id as usize)
1294 .is_some_and(Option::is_some)
1295 {
1296 return Err(invalid_admission(
1297 "sequence authority allocator selected a live sparse id",
1298 ));
1299 }
1300 Ok(SequenceAuthorityReservation {
1301 authority: SequenceAuthorityId {
1302 sparse_id,
1303 generation,
1304 },
1305 source,
1306 next_generation,
1307 })
1308 }
1309
1310 fn prepare_sequence_storage(
1311 &mut self,
1312 reservation: SequenceAuthorityReservation,
1313 ) -> Result<(), VNextError> {
1314 if matches!(reservation.source, SparseIdSource::Fresh) {
1315 self.live_sequences.try_reserve(1).map_err(|_| {
1316 admission_fault(
1317 DynamicAdmissionFaultKind::AllocationFailure,
1318 "cannot reserve sequence authority slab",
1319 )
1320 })?;
1321 self.reusable_sequence_ids
1322 .try_reserve(self.live_sequences.len() + 1)
1323 .map_err(|_| {
1324 admission_fault(
1325 DynamicAdmissionFaultKind::AllocationFailure,
1326 "cannot reserve sequence release free-list",
1327 )
1328 })?;
1329 }
1330 Ok(())
1331 }
1332
1333 fn commit_sequence_authority(
1334 &mut self,
1335 reservation: SequenceAuthorityReservation,
1336 request: RequestAuthorityId,
1337 ) -> SequenceAuthorityId {
1338 match reservation.source {
1339 SparseIdSource::Reused => {
1340 self.reusable_sequence_ids.pop();
1341 self.live_sequences[reservation.authority.sparse_id as usize] =
1342 Some(LiveSequenceRecord {
1343 generation: reservation.authority.generation,
1344 request,
1345 active_child_claims: 0,
1346 });
1347 }
1348 SparseIdSource::Fresh => {
1349 self.live_sequences.push(Some(LiveSequenceRecord {
1350 generation: reservation.authority.generation,
1351 request,
1352 active_child_claims: 0,
1353 }));
1354 }
1355 }
1356 self.next_sequence_generation = reservation.next_generation;
1357 reservation.authority
1358 }
1359}
1360
1361#[derive(Debug, Clone, Copy)]
1362enum SparseIdSource {
1363 Reused,
1364 Fresh,
1365}
1366
1367#[derive(Debug, Clone, Copy)]
1368struct RequestAuthorityReservation {
1369 authority: RequestAuthorityId,
1370 source: SparseIdSource,
1371 next_generation: u64,
1372}
1373
1374#[derive(Debug, Clone, Copy)]
1375struct SequenceAuthorityReservation {
1376 authority: SequenceAuthorityId,
1377 source: SparseIdSource,
1378 next_generation: u64,
1379}
1380
1381#[derive(Debug)]
1382struct CoordinatorInner {
1383 id: LogicalAdmissionCoordinatorId,
1384 state: Mutex<CoordinatorState>,
1385 epoch_tx: watch::Sender<CapacityEpochs>,
1386}
1387
1388impl CoordinatorInner {
1389 fn lock_state(&self) -> Result<MutexGuard<'_, CoordinatorState>, VNextError> {
1390 match self.state.lock() {
1391 Ok(state) => Ok(state),
1392 Err(poisoned) => {
1393 let mut state = poisoned.into_inner();
1394 state.poisoned = true;
1395 let epochs = state.epochs(self.id);
1396 self.epoch_tx.send_replace(epochs);
1397 Err(admission_fault(
1398 DynamicAdmissionFaultKind::Poisoned,
1399 "coordinator state is poisoned",
1400 ))
1401 }
1402 }
1403 }
1404
1405 fn lock_mutation(&self) -> Result<CoordinatorMutationGuard<'_>, VNextError> {
1406 Ok(CoordinatorMutationGuard {
1407 inner: self,
1408 state: self.lock_state()?,
1409 panicking_on_entry: std::thread::panicking(),
1410 })
1411 }
1412}
1413
1414struct CoordinatorMutationGuard<'a> {
1415 inner: &'a CoordinatorInner,
1416 state: MutexGuard<'a, CoordinatorState>,
1417 panicking_on_entry: bool,
1418}
1419
1420impl Deref for CoordinatorMutationGuard<'_> {
1421 type Target = CoordinatorState;
1422
1423 fn deref(&self) -> &Self::Target {
1424 &self.state
1425 }
1426}
1427
1428impl DerefMut for CoordinatorMutationGuard<'_> {
1429 fn deref_mut(&mut self) -> &mut Self::Target {
1430 &mut self.state
1431 }
1432}
1433
1434impl Drop for CoordinatorMutationGuard<'_> {
1435 fn drop(&mut self) {
1436 if !self.panicking_on_entry && std::thread::panicking() {
1437 self.state.poisoned = true;
1438 self.inner
1439 .epoch_tx
1440 .send_replace(self.state.epochs(self.inner.id));
1441 }
1442 }
1443}
1444
1445#[derive(Debug, Clone)]
1446pub struct LogicalAdmissionCoordinator {
1447 inner: Arc<CoordinatorInner>,
1448}
1449
1450impl LogicalAdmissionCoordinator {
1451 pub(crate) fn new(
1452 domains: Vec<(CapacityDomainId, CapacityDomainSpec)>,
1453 maximum_active_sequences: u32,
1454 ) -> Result<Self, VNextError> {
1455 Self::with_checkpoint_capacity(domains, maximum_active_sequences, None)
1456 }
1457
1458 pub(crate) fn with_checkpoint_capacity(
1461 domains: Vec<(CapacityDomainId, CapacityDomainSpec)>,
1462 maximum_active_sequences: u32,
1463 checkpoint_capacity: Option<CheckpointCapacityPolicy>,
1464 ) -> Result<Self, VNextError> {
1465 if maximum_active_sequences == 0 {
1466 return Err(invalid_admission(
1467 "coordinator requires a non-zero sequence ceiling",
1468 ));
1469 }
1470 let mut registered = BTreeMap::new();
1471 for (domain, spec) in domains {
1472 if registered
1473 .insert(
1474 domain,
1475 DomainState {
1476 spec,
1477 used: 0,
1478 availability_epoch: 1,
1479 },
1480 )
1481 .is_some()
1482 {
1483 return Err(invalid_admission("duplicate coordinator capacity domain"));
1484 }
1485 }
1486 let coordinator_id = LogicalAdmissionCoordinatorId::issue()?;
1487 let initial_epochs = CapacityEpochs {
1488 coordinator_id,
1489 release_epoch: 1,
1490 capacity_epoch: 1,
1491 };
1492 let (epoch_tx, _) = watch::channel(initial_epochs);
1493 Ok(Self {
1494 inner: Arc::new(CoordinatorInner {
1495 id: coordinator_id,
1496 state: Mutex::new(CoordinatorState {
1497 domains: registered,
1498 maximum_active_sequences,
1499 active_requests: 0,
1500 active_sequences: 0,
1501 active_child_claims: 0,
1502 checkpoint_claims: CheckpointClaimLedger::new(checkpoint_capacity),
1503 live_requests: Vec::new(),
1504 reusable_request_ids: Vec::new(),
1505 live_sequences: Vec::new(),
1506 reusable_sequence_ids: Vec::new(),
1507 next_request_generation: 1,
1508 next_sequence_generation: 1,
1509 release_epoch: 1,
1510 capacity_epoch: 1,
1511 active_sequence_availability_epoch: 1,
1512 poisoned: false,
1513 }),
1514 epoch_tx,
1515 }),
1516 })
1517 }
1518
1519 pub fn id(&self) -> LogicalAdmissionCoordinatorId {
1520 self.inner.id
1521 }
1522
1523 pub(crate) fn owns(&self, lease: &LogicalAdmissionLease) -> bool {
1524 self.id() == lease.coordinator_id() && Arc::ptr_eq(&self.inner, &lease.inner)
1525 }
1526
1527 pub(crate) fn owns_request(&self, lease: &LogicalRequestLease) -> bool {
1528 self.id() == lease.coordinator_id() && Arc::ptr_eq(&self.inner, &lease.inner)
1529 }
1530
1531 pub(crate) fn try_admit_request(
1532 &self,
1533 demand: &AdmissionDemand,
1534 ) -> Result<RequestAdmissionDecision, VNextError> {
1535 let mut state = self.inner.lock_mutation()?;
1536 if state.poisoned {
1537 return Err(admission_fault(
1538 DynamicAdmissionFaultKind::Poisoned,
1539 "coordinator is fail-closed",
1540 ));
1541 }
1542
1543 let evaluation = evaluate_demand(&state, demand)?;
1544 if !evaluation.permanent.is_empty() {
1545 return Ok(RequestAdmissionDecision::PermanentRejected(
1546 AdmissionRejected {
1547 immediate_requested: demand.immediate_claim.clone(),
1548 fit_requested: demand.fit_requirement.clone(),
1549 maximum: state.snapshot(self.id()),
1550 blockers: evaluation.permanent,
1551 },
1552 ));
1553 }
1554 if !evaluation.blockers.is_empty() {
1555 let action = deferred_action(demand, evaluation.growth_required);
1556 let wait_condition =
1557 state.wait_condition_for_blockers(self.id(), &evaluation.blockers)?;
1558 let snapshot = state.snapshot(self.id());
1559 return Ok(RequestAdmissionDecision::Deferred(AdmissionDeferred {
1560 immediate_requested: demand.immediate_claim.clone(),
1561 fit_requested: demand.fit_requirement.clone(),
1562 release_epoch: snapshot.release_epoch,
1563 capacity_epoch: snapshot.capacity_epoch,
1564 available: snapshot,
1565 blockers: evaluation.blockers,
1566 action,
1567 wait_condition,
1568 }));
1569 }
1570
1571 let request = state.preview_request_authority()?;
1572 let committed_claims = demand.immediate_claim.clone();
1573 let mut next_usage = Vec::with_capacity(demand.immediate_claim.entries().len());
1574 for entry in demand.immediate_claim.entries() {
1575 let domain = state
1576 .domains
1577 .get(&entry.domain)
1578 .expect("known request demand domain was preflight validated");
1579 let used = domain.used.checked_add(entry.units.get()).ok_or_else(|| {
1580 admission_fault(
1581 DynamicAdmissionFaultKind::ArithmeticOverflow,
1582 "request capacity usage overflows u64",
1583 )
1584 })?;
1585 next_usage.push((entry.domain, used));
1586 }
1587 let next_active_requests = state.active_requests.checked_add(1).ok_or_else(|| {
1588 admission_fault(
1589 DynamicAdmissionFaultKind::AuthorityExhausted,
1590 "active request count is exhausted",
1591 )
1592 })?;
1593 state
1594 .release_epoch
1595 .checked_add(u64::from(next_active_requests))
1596 .and_then(|epoch| epoch.checked_add(u64::from(state.active_sequences)))
1597 .and_then(|epoch| epoch.checked_add(state.active_child_claims))
1598 .and_then(|epoch| epoch.checked_add(state.checkpoint_claims.count()))
1599 .ok_or_else(|| {
1600 admission_fault(
1601 DynamicAdmissionFaultKind::EpochExhausted,
1602 "release epoch cannot represent every outstanding lease release",
1603 )
1604 })?;
1605 state.prepare_request_storage(request)?;
1606 let request = state.commit_request_authority(request);
1607 for (domain, used) in next_usage {
1608 state
1609 .domains
1610 .get_mut(&domain)
1611 .expect("validated request capacity domain remains registered")
1612 .used = used;
1613 }
1614 state.active_requests = next_active_requests;
1615 Ok(RequestAdmissionDecision::Admitted(LogicalRequestLease {
1616 inner: Arc::clone(&self.inner),
1617 request,
1618 claims: committed_claims,
1619 released: false,
1620 }))
1621 }
1622
1623 pub(crate) fn preflight_initial_sequence(
1629 &self,
1630 request: &AdmissionDemand,
1631 sequence: &AdmissionDemand,
1632 ) -> Result<AdmissionPreflightDecision, VNextError> {
1633 let demand = AdmissionDemand::initial_sequence_bundle(request, sequence)?;
1634 let state = self.inner.lock_state()?;
1635 if state.poisoned {
1636 return Err(admission_fault(
1637 DynamicAdmissionFaultKind::Poisoned,
1638 "coordinator is fail-closed",
1639 ));
1640 }
1641 let mut evaluation = evaluate_demand(&state, &demand)?;
1642 if !evaluation.permanent.is_empty() {
1643 return Ok(AdmissionPreflightDecision::PermanentRejected(
1644 AdmissionRejected {
1645 immediate_requested: demand.immediate_claim,
1646 fit_requested: demand.fit_requirement,
1647 maximum: state.snapshot(self.id()),
1648 blockers: evaluation.permanent,
1649 },
1650 ));
1651 }
1652 if state.active_sequences < state.maximum_active_sequences {
1653 return Ok(AdmissionPreflightDecision::Eligible);
1654 }
1655 evaluation.blockers.push(CapacityShortfall {
1656 domain: None,
1657 kind: CapacityShortfallKind::ActiveSequenceCeiling,
1658 requested: CapacityUnits::new(1),
1659 available: CapacityUnits::ZERO,
1660 current_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
1661 maximum_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
1662 });
1663 let wait_condition = state.wait_condition_for_blockers(self.id(), &evaluation.blockers)?;
1664 let snapshot = state.snapshot(self.id());
1665 Ok(AdmissionPreflightDecision::Deferred(AdmissionDeferred {
1666 immediate_requested: demand.immediate_claim,
1667 fit_requested: demand.fit_requirement,
1668 release_epoch: snapshot.release_epoch,
1669 capacity_epoch: snapshot.capacity_epoch,
1670 available: snapshot,
1671 blockers: evaluation.blockers,
1672 action: match demand.pressure_action {
1673 AdmissionPressureAction::WaitForRelease => DeferredAction::WaitForRelease,
1674 AdmissionPressureAction::PreemptAndRecompute => DeferredAction::PreemptAndRecompute,
1675 },
1676 wait_condition,
1677 }))
1678 }
1679
1680 pub(crate) fn observe_initial_sequence(
1684 &self,
1685 request: &AdmissionDemand,
1686 sequence: &AdmissionDemand,
1687 ) -> Result<AdmissionPreflightDecision, VNextError> {
1688 let demand = AdmissionDemand::initial_sequence_bundle(request, sequence)?;
1689 let state = self.inner.lock_state()?;
1690 if state.poisoned {
1691 return Err(admission_fault(
1692 DynamicAdmissionFaultKind::Poisoned,
1693 "coordinator is fail-closed",
1694 ));
1695 }
1696 let mut evaluation = evaluate_demand(&state, &demand)?;
1697 if !evaluation.permanent.is_empty() {
1698 return Ok(AdmissionPreflightDecision::PermanentRejected(
1699 AdmissionRejected {
1700 immediate_requested: demand.immediate_claim,
1701 fit_requested: demand.fit_requirement,
1702 maximum: state.snapshot(self.id()),
1703 blockers: evaluation.permanent,
1704 },
1705 ));
1706 }
1707 if state.active_sequences >= state.maximum_active_sequences {
1708 evaluation.blockers.push(CapacityShortfall {
1709 domain: None,
1710 kind: CapacityShortfallKind::ActiveSequenceCeiling,
1711 requested: CapacityUnits::new(1),
1712 available: CapacityUnits::ZERO,
1713 current_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
1714 maximum_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
1715 });
1716 }
1717 if evaluation.blockers.is_empty() {
1718 return Ok(AdmissionPreflightDecision::Eligible);
1719 }
1720 let action = deferred_action(&demand, evaluation.growth_required);
1721 let wait_condition = state.wait_condition_for_blockers(self.id(), &evaluation.blockers)?;
1722 let snapshot = state.snapshot(self.id());
1723 Ok(AdmissionPreflightDecision::Deferred(AdmissionDeferred {
1724 immediate_requested: demand.immediate_claim,
1725 fit_requested: demand.fit_requirement,
1726 release_epoch: snapshot.release_epoch,
1727 capacity_epoch: snapshot.capacity_epoch,
1728 available: snapshot,
1729 blockers: evaluation.blockers,
1730 action,
1731 wait_condition,
1732 }))
1733 }
1734
1735 pub(crate) fn try_admit_initial_sequence(
1739 &self,
1740 request_demand: &AdmissionDemand,
1741 sequence_demand: &AdmissionDemand,
1742 ) -> Result<InitialSequenceAdmissionDecision, VNextError> {
1743 let demand = AdmissionDemand::initial_sequence_bundle(request_demand, sequence_demand)?;
1744 let mut state = self.inner.lock_mutation()?;
1745 if state.poisoned {
1746 return Err(admission_fault(
1747 DynamicAdmissionFaultKind::Poisoned,
1748 "coordinator is fail-closed",
1749 ));
1750 }
1751
1752 let mut evaluation = evaluate_demand(&state, &demand)?;
1753 if !evaluation.permanent.is_empty() {
1754 return Ok(InitialSequenceAdmissionDecision::PermanentRejected(
1755 AdmissionRejected {
1756 immediate_requested: demand.immediate_claim,
1757 fit_requested: demand.fit_requirement,
1758 maximum: state.snapshot(self.id()),
1759 blockers: evaluation.permanent,
1760 },
1761 ));
1762 }
1763 if state.active_sequences >= state.maximum_active_sequences {
1764 evaluation.blockers.push(CapacityShortfall {
1765 domain: None,
1766 kind: CapacityShortfallKind::ActiveSequenceCeiling,
1767 requested: CapacityUnits::new(1),
1768 available: CapacityUnits::ZERO,
1769 current_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
1770 maximum_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
1771 });
1772 }
1773 if !evaluation.blockers.is_empty() {
1774 return Ok(InitialSequenceAdmissionDecision::Deferred);
1775 }
1776
1777 let request_reservation = state.preview_request_authority()?;
1778 let sequence_reservation = state.preview_sequence_authority()?;
1779 let mut next_usage = Vec::with_capacity(demand.immediate_claim.entries().len());
1780 for entry in demand.immediate_claim.entries() {
1781 let domain = state
1782 .domains
1783 .get(&entry.domain)
1784 .expect("known bundle demand domain was preflight validated");
1785 let used = domain.used.checked_add(entry.units.get()).ok_or_else(|| {
1786 admission_fault(
1787 DynamicAdmissionFaultKind::ArithmeticOverflow,
1788 "initial request/sequence capacity usage overflows u64",
1789 )
1790 })?;
1791 next_usage.push((entry.domain, used));
1792 }
1793 let next_active_requests = state.active_requests.checked_add(1).ok_or_else(|| {
1794 admission_fault(
1795 DynamicAdmissionFaultKind::AuthorityExhausted,
1796 "active request count is exhausted",
1797 )
1798 })?;
1799 let next_active_sequences = state.active_sequences.checked_add(1).ok_or_else(|| {
1800 admission_fault(
1801 DynamicAdmissionFaultKind::ArithmeticOverflow,
1802 "active sequence count overflows u32",
1803 )
1804 })?;
1805 state
1806 .release_epoch
1807 .checked_add(u64::from(next_active_requests))
1808 .and_then(|epoch| epoch.checked_add(u64::from(next_active_sequences)))
1809 .and_then(|epoch| epoch.checked_add(state.active_child_claims))
1810 .and_then(|epoch| epoch.checked_add(state.checkpoint_claims.count()))
1811 .ok_or_else(|| {
1812 admission_fault(
1813 DynamicAdmissionFaultKind::EpochExhausted,
1814 "release epoch cannot represent the initial request/sequence bundle",
1815 )
1816 })?;
1817 state.prepare_request_storage(request_reservation)?;
1818 state.prepare_sequence_storage(sequence_reservation)?;
1819
1820 let request = state.commit_request_authority(request_reservation);
1821 let sequence = state.commit_sequence_authority(sequence_reservation, request);
1822 for (domain, used) in next_usage {
1823 state
1824 .domains
1825 .get_mut(&domain)
1826 .expect("validated bundle capacity domain remains registered")
1827 .used = used;
1828 }
1829 state.active_requests = next_active_requests;
1830 state.active_sequences = next_active_sequences;
1831 state.live_requests[request.sparse_id as usize]
1832 .as_mut()
1833 .expect("new initial request remains live")
1834 .active_sequences = 1;
1835
1836 Ok(InitialSequenceAdmissionDecision::Admitted(
1837 LogicalInitialSequenceAdmission {
1838 sequence: LogicalAdmissionLease {
1839 inner: Arc::clone(&self.inner),
1840 request,
1841 sequence,
1842 claims: sequence_demand.immediate_claim.clone(),
1843 released: false,
1844 },
1845 request: LogicalRequestLease {
1846 inner: Arc::clone(&self.inner),
1847 request,
1848 claims: request_demand.immediate_claim.clone(),
1849 released: false,
1850 },
1851 },
1852 ))
1853 }
1854
1855 pub(crate) fn try_admit_sequence_for_request(
1856 &self,
1857 request: &LogicalRequestLease,
1858 demand: &AdmissionDemand,
1859 ) -> Result<AdmissionDecision, VNextError> {
1860 if !self.owns_request(request) {
1861 return Err(admission_fault(
1862 DynamicAdmissionFaultKind::ForeignCoordinator,
1863 "parent request belongs to another coordinator",
1864 ));
1865 }
1866 if request.released {
1867 return Err(invalid_admission(
1868 "sequence admission requires a live parent request",
1869 ));
1870 }
1871 let mut state = self.inner.lock_mutation()?;
1872 if state.poisoned {
1873 return Err(admission_fault(
1874 DynamicAdmissionFaultKind::Poisoned,
1875 "coordinator is fail-closed",
1876 ));
1877 }
1878 let request_record = state
1879 .live_requests
1880 .get(request.request.sparse_id as usize)
1881 .and_then(|record| *record)
1882 .filter(|record| record.generation == request.request.generation)
1883 .ok_or_else(|| invalid_admission("parent request authority is stale or not live"))?;
1884
1885 let mut evaluation = evaluate_demand(&state, demand)?;
1886 if !evaluation.permanent.is_empty() {
1887 return Ok(AdmissionDecision::PermanentRejected(AdmissionRejected {
1888 immediate_requested: demand.immediate_claim.clone(),
1889 fit_requested: demand.fit_requirement.clone(),
1890 maximum: state.snapshot(self.id()),
1891 blockers: evaluation.permanent,
1892 }));
1893 }
1894 if state.active_sequences >= state.maximum_active_sequences {
1895 evaluation.blockers.push(CapacityShortfall {
1896 domain: None,
1897 kind: CapacityShortfallKind::ActiveSequenceCeiling,
1898 requested: CapacityUnits::new(1),
1899 available: CapacityUnits::ZERO,
1900 current_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
1901 maximum_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
1902 });
1903 }
1904 if !evaluation.blockers.is_empty() {
1905 let action = deferred_action(demand, evaluation.growth_required);
1906 let wait_condition =
1907 state.wait_condition_for_blockers(self.id(), &evaluation.blockers)?;
1908 let snapshot = state.snapshot(self.id());
1909 return Ok(AdmissionDecision::Deferred(AdmissionDeferred {
1910 immediate_requested: demand.immediate_claim.clone(),
1911 fit_requested: demand.fit_requirement.clone(),
1912 release_epoch: snapshot.release_epoch,
1913 capacity_epoch: snapshot.capacity_epoch,
1914 available: snapshot,
1915 blockers: evaluation.blockers,
1916 action,
1917 wait_condition,
1918 }));
1919 }
1920
1921 let sequence = state.preview_sequence_authority()?;
1922 let committed_claims = demand.immediate_claim.clone();
1923 let mut next_usage = Vec::with_capacity(demand.immediate_claim.entries().len());
1924 for entry in demand.immediate_claim.entries() {
1925 let domain = state
1926 .domains
1927 .get(&entry.domain)
1928 .expect("known demand domain was preflight validated");
1929 let used = domain.used.checked_add(entry.units.get()).ok_or_else(|| {
1930 admission_fault(
1931 DynamicAdmissionFaultKind::ArithmeticOverflow,
1932 "capacity usage overflows u64",
1933 )
1934 })?;
1935 next_usage.push((entry.domain, used));
1936 }
1937 let next_active_sequences = state.active_sequences.checked_add(1).ok_or_else(|| {
1938 admission_fault(
1939 DynamicAdmissionFaultKind::ArithmeticOverflow,
1940 "active sequence count overflows u32",
1941 )
1942 })?;
1943 let next_request_sequences =
1944 request_record
1945 .active_sequences
1946 .checked_add(1)
1947 .ok_or_else(|| {
1948 admission_fault(
1949 DynamicAdmissionFaultKind::AuthorityExhausted,
1950 "request child sequence count is exhausted",
1951 )
1952 })?;
1953 state
1954 .release_epoch
1955 .checked_add(u64::from(state.active_requests))
1956 .and_then(|epoch| epoch.checked_add(u64::from(next_active_sequences)))
1957 .and_then(|epoch| epoch.checked_add(state.active_child_claims))
1958 .and_then(|epoch| epoch.checked_add(state.checkpoint_claims.count()))
1959 .ok_or_else(|| {
1960 admission_fault(
1961 DynamicAdmissionFaultKind::EpochExhausted,
1962 "release epoch cannot represent every outstanding lease release",
1963 )
1964 })?;
1965 state.prepare_sequence_storage(sequence)?;
1966 let sequence = state.commit_sequence_authority(sequence, request.request);
1967 for (domain, used) in next_usage {
1968 state
1969 .domains
1970 .get_mut(&domain)
1971 .expect("validated capacity domain remains registered")
1972 .used = used;
1973 }
1974 state.active_sequences = next_active_sequences;
1975 state.live_requests[request.request.sparse_id as usize]
1976 .as_mut()
1977 .expect("validated parent request remains live")
1978 .active_sequences = next_request_sequences;
1979 Ok(AdmissionDecision::Admitted(LogicalAdmissionLease {
1980 inner: Arc::clone(&self.inner),
1981 request: request.request,
1982 sequence,
1983 claims: committed_claims,
1984 released: false,
1985 }))
1986 }
1987
1988 pub(crate) fn preflight_sequence_ceiling_for_request(
1993 &self,
1994 request: &LogicalRequestLease,
1995 demand: &AdmissionDemand,
1996 ) -> Result<AdmissionPreflightDecision, VNextError> {
1997 if !self.owns_request(request) {
1998 return Err(admission_fault(
1999 DynamicAdmissionFaultKind::ForeignCoordinator,
2000 "parent request belongs to another coordinator",
2001 ));
2002 }
2003 if request.released {
2004 return Err(invalid_admission(
2005 "sequence admission requires a live parent request",
2006 ));
2007 }
2008 let state = self.inner.lock_state()?;
2009 if state.poisoned {
2010 return Err(admission_fault(
2011 DynamicAdmissionFaultKind::Poisoned,
2012 "coordinator is fail-closed",
2013 ));
2014 }
2015 state
2016 .live_requests
2017 .get(request.request.sparse_id as usize)
2018 .and_then(|record| *record)
2019 .filter(|record| record.generation == request.request.generation)
2020 .ok_or_else(|| invalid_admission("parent request authority is stale or not live"))?;
2021
2022 let mut evaluation = evaluate_demand(&state, demand)?;
2023 if !evaluation.permanent.is_empty() {
2024 return Ok(AdmissionPreflightDecision::PermanentRejected(
2025 AdmissionRejected {
2026 immediate_requested: demand.immediate_claim.clone(),
2027 fit_requested: demand.fit_requirement.clone(),
2028 maximum: state.snapshot(self.id()),
2029 blockers: evaluation.permanent,
2030 },
2031 ));
2032 }
2033 if state.active_sequences < state.maximum_active_sequences {
2034 return Ok(AdmissionPreflightDecision::Eligible);
2035 }
2036 evaluation.blockers.push(CapacityShortfall {
2037 domain: None,
2038 kind: CapacityShortfallKind::ActiveSequenceCeiling,
2039 requested: CapacityUnits::new(1),
2040 available: CapacityUnits::ZERO,
2041 current_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
2042 maximum_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
2043 });
2044 let wait_condition = state.wait_condition_for_blockers(self.id(), &evaluation.blockers)?;
2045 let snapshot = state.snapshot(self.id());
2046 Ok(AdmissionPreflightDecision::Deferred(AdmissionDeferred {
2047 immediate_requested: demand.immediate_claim.clone(),
2048 fit_requested: demand.fit_requirement.clone(),
2049 release_epoch: snapshot.release_epoch,
2050 capacity_epoch: snapshot.capacity_epoch,
2051 available: snapshot,
2052 blockers: evaluation.blockers,
2053 action: match demand.pressure_action {
2054 AdmissionPressureAction::WaitForRelease => DeferredAction::WaitForRelease,
2055 AdmissionPressureAction::PreemptAndRecompute => DeferredAction::PreemptAndRecompute,
2056 },
2057 wait_condition,
2058 }))
2059 }
2060
2061 pub(crate) fn try_claim_for_sequence(
2062 &self,
2063 sequence: &LogicalAdmissionLease,
2064 demand: &AdmissionDemand,
2065 ) -> Result<CapacityClaimDecision, VNextError> {
2066 match self.try_claim_for_sequences(&[sequence], demand)? {
2067 BatchCapacityClaimDecision::Claimed(batch) => {
2068 Ok(CapacityClaimDecision::Claimed(LogicalCapacityLease {
2069 batch,
2070 }))
2071 }
2072 BatchCapacityClaimDecision::Deferred(deferred) => {
2073 Ok(CapacityClaimDecision::Deferred(deferred))
2074 }
2075 BatchCapacityClaimDecision::PermanentRejected(rejected) => {
2076 Ok(CapacityClaimDecision::PermanentRejected(rejected))
2077 }
2078 }
2079 }
2080
2081 pub(crate) fn try_claim_for_sequences(
2086 &self,
2087 sequences: &[&LogicalAdmissionLease],
2088 demand: &AdmissionDemand,
2089 ) -> Result<BatchCapacityClaimDecision, VNextError> {
2090 if sequences.is_empty() || demand.immediate_claim.is_empty() {
2091 return Err(invalid_admission(
2092 "batch child capacity claim requires live parents and non-empty demand",
2093 ));
2094 }
2095 let mut parents = Vec::with_capacity(sequences.len());
2096 for sequence in sequences {
2097 if !self.owns(sequence) {
2098 return Err(admission_fault(
2099 DynamicAdmissionFaultKind::ForeignCoordinator,
2100 "batch parent sequence belongs to another coordinator",
2101 ));
2102 }
2103 if sequence.released {
2104 return Err(invalid_admission(
2105 "batch child capacity claim requires every parent to be live",
2106 ));
2107 }
2108 parents.push(SequenceCapacityParent {
2109 request: sequence.request,
2110 sequence: sequence.sequence,
2111 });
2112 }
2113 u32::try_from(parents.len())
2114 .map_err(|_| invalid_admission("batch parent count exceeds the protocol range"))?;
2115 parents.sort_by_key(|parent| (parent.sequence, parent.request));
2116 if parents.windows(2).any(|pair| pair[0] == pair[1]) {
2117 return Err(invalid_admission(
2118 "batch child capacity claim contains a duplicate parent sequence",
2119 ));
2120 }
2121
2122 let mut state = self.inner.lock_mutation()?;
2123 if state.poisoned {
2124 return Err(admission_fault(
2125 DynamicAdmissionFaultKind::Poisoned,
2126 "coordinator is fail-closed",
2127 ));
2128 }
2129 let mut next_parent_child_claims = Vec::with_capacity(parents.len());
2130 for parent in &parents {
2131 let record = state
2132 .live_sequences
2133 .get(parent.sequence.sparse_id as usize)
2134 .and_then(|record| *record)
2135 .filter(|record| {
2136 record.generation == parent.sequence.generation
2137 && record.request == parent.request
2138 })
2139 .ok_or_else(|| {
2140 invalid_admission("batch parent sequence authority is stale or not live")
2141 })?;
2142 let next = record.active_child_claims.checked_add(1).ok_or_else(|| {
2143 admission_fault(
2144 DynamicAdmissionFaultKind::AuthorityExhausted,
2145 "batch parent child capacity claim count is exhausted",
2146 )
2147 })?;
2148 next_parent_child_claims.push((parent.sequence.sparse_id, next));
2149 }
2150
2151 let evaluation = evaluate_demand(&state, demand)?;
2152 if !evaluation.permanent.is_empty() {
2153 return Ok(BatchCapacityClaimDecision::PermanentRejected(
2154 AdmissionRejected {
2155 immediate_requested: demand.immediate_claim.clone(),
2156 fit_requested: demand.fit_requirement.clone(),
2157 maximum: state.snapshot(self.id()),
2158 blockers: evaluation.permanent,
2159 },
2160 ));
2161 }
2162 if !evaluation.blockers.is_empty() {
2163 let action = deferred_action(demand, evaluation.growth_required);
2164 let wait_condition =
2165 state.wait_condition_for_blockers(self.id(), &evaluation.blockers)?;
2166 let snapshot = state.snapshot(self.id());
2167 return Ok(BatchCapacityClaimDecision::Deferred(AdmissionDeferred {
2168 immediate_requested: demand.immediate_claim.clone(),
2169 fit_requested: demand.fit_requirement.clone(),
2170 release_epoch: snapshot.release_epoch,
2171 capacity_epoch: snapshot.capacity_epoch,
2172 available: snapshot,
2173 blockers: evaluation.blockers,
2174 action,
2175 wait_condition,
2176 }));
2177 }
2178
2179 let committed_claims = demand.immediate_claim.clone();
2180 let mut next_usage = Vec::with_capacity(demand.immediate_claim.entries().len());
2181 for entry in demand.immediate_claim.entries() {
2182 let domain = state
2183 .domains
2184 .get(&entry.domain)
2185 .expect("known batch demand domain was preflight validated");
2186 let used = domain.used.checked_add(entry.units.get()).ok_or_else(|| {
2187 admission_fault(
2188 DynamicAdmissionFaultKind::ArithmeticOverflow,
2189 "batch child capacity usage overflows u64",
2190 )
2191 })?;
2192 next_usage.push((entry.domain, used));
2193 }
2194 let next_child_claims = state.active_child_claims.checked_add(1).ok_or_else(|| {
2195 admission_fault(
2196 DynamicAdmissionFaultKind::AuthorityExhausted,
2197 "active batch child capacity claim count is exhausted",
2198 )
2199 })?;
2200 state
2201 .release_epoch
2202 .checked_add(u64::from(state.active_requests))
2203 .and_then(|epoch| epoch.checked_add(u64::from(state.active_sequences)))
2204 .and_then(|epoch| epoch.checked_add(next_child_claims))
2205 .and_then(|epoch| epoch.checked_add(state.checkpoint_claims.count()))
2206 .ok_or_else(|| {
2207 admission_fault(
2208 DynamicAdmissionFaultKind::EpochExhausted,
2209 "release epoch cannot represent every outstanding lease release",
2210 )
2211 })?;
2212
2213 for (domain, used) in next_usage {
2214 state
2215 .domains
2216 .get_mut(&domain)
2217 .expect("validated batch capacity domain remains registered")
2218 .used = used;
2219 }
2220 state.active_child_claims = next_child_claims;
2221 for (sparse_id, next) in next_parent_child_claims {
2222 state.live_sequences[sparse_id as usize]
2223 .as_mut()
2224 .expect("validated batch parent sequence remains live")
2225 .active_child_claims = next;
2226 }
2227 Ok(BatchCapacityClaimDecision::Claimed(
2228 LogicalBatchCapacityLease {
2229 inner: Arc::clone(&self.inner),
2230 parents,
2231 claims: committed_claims,
2232 released: false,
2233 },
2234 ))
2235 }
2236
2237 pub(crate) fn owns_capacity_claim(&self, lease: &LogicalCapacityLease) -> bool {
2238 self.owns_batch_capacity_claim(&lease.batch)
2239 }
2240
2241 pub(crate) fn owns_batch_capacity_claim(&self, lease: &LogicalBatchCapacityLease) -> bool {
2242 self.id() == lease.coordinator_id() && Arc::ptr_eq(&self.inner, &lease.inner)
2243 }
2244
2245 pub fn snapshot(&self) -> Result<CapacitySnapshot, VNextError> {
2246 match self.inner.state.lock() {
2247 Ok(state) => Ok(state.snapshot(self.id())),
2248 Err(poisoned) => {
2249 let mut state = poisoned.into_inner();
2250 state.poisoned = true;
2251 let snapshot = state.snapshot(self.id());
2252 self.inner.epoch_tx.send_replace(state.epochs(self.id()));
2253 Ok(snapshot)
2254 }
2255 }
2256 }
2257
2258 pub fn epochs(&self) -> Result<CapacityEpochs, VNextError> {
2259 let state = self.inner.lock_state()?;
2260 if state.poisoned {
2261 return Err(admission_fault(
2262 DynamicAdmissionFaultKind::Poisoned,
2263 "coordinator is fail-closed",
2264 ));
2265 }
2266 Ok(state.epochs(self.id()))
2267 }
2268
2269 pub(crate) fn subscribe_epochs(&self) -> watch::Receiver<CapacityEpochs> {
2270 self.inner.epoch_tx.subscribe()
2271 }
2272
2273 pub fn write_availability_epochs(
2276 &self,
2277 out: &mut Vec<CapacityAvailabilityEpoch>,
2278 ) -> Result<CapacityEpochs, VNextError> {
2279 let state = self.inner.lock_state()?;
2280 if state.poisoned {
2281 return Err(admission_fault(
2282 DynamicAdmissionFaultKind::Poisoned,
2283 "coordinator is fail-closed",
2284 ));
2285 }
2286 state.write_availability_epochs(out);
2287 Ok(state.epochs(self.id()))
2288 }
2289
2290 pub(crate) fn wait_snapshot_for_domains(
2291 &self,
2292 domains: impl IntoIterator<Item = CapacityDomainId>,
2293 ) -> Result<CapacityWaitSnapshot, VNextError> {
2294 let state = self.inner.lock_state()?;
2295 if state.poisoned {
2296 return Err(admission_fault(
2297 DynamicAdmissionFaultKind::Poisoned,
2298 "coordinator is fail-closed",
2299 ));
2300 }
2301 Ok(CapacityWaitSnapshot::new(
2302 state.epochs(self.id()),
2303 state.wait_condition_for_domains(self.id(), domains)?,
2304 ))
2305 }
2306
2307 pub(crate) fn refresh_wait_snapshot(
2308 &self,
2309 observed: &CapacityWaitCondition,
2310 ) -> Result<CapacityWaitSnapshot, VNextError> {
2311 let state = self.inner.lock_state()?;
2312 if state.poisoned {
2313 return Err(admission_fault(
2314 DynamicAdmissionFaultKind::Poisoned,
2315 "coordinator is fail-closed",
2316 ));
2317 }
2318 Ok(CapacityWaitSnapshot::new(
2319 state.epochs(self.id()),
2320 state.refresh_wait_condition(self.id(), observed)?,
2321 ))
2322 }
2323
2324 pub(crate) fn set_domain_total(
2325 &self,
2326 domain: CapacityDomainId,
2327 new_total: CapacityUnits,
2328 ) -> Result<CapacityEpochs, VNextError> {
2329 self.set_domain_totals(&[(domain, new_total)])
2330 }
2331
2332 pub(crate) fn set_domain_totals(
2333 &self,
2334 updates: &[(CapacityDomainId, CapacityUnits)],
2335 ) -> Result<CapacityEpochs, VNextError> {
2336 let mut state = self.inner.lock_mutation()?;
2337 if state.poisoned {
2338 return Err(admission_fault(
2339 DynamicAdmissionFaultKind::Poisoned,
2340 "coordinator is fail-closed",
2341 ));
2342 }
2343 let mut seen = BTreeSet::new();
2344 let mut changed = false;
2345 for (domain, new_total) in updates {
2346 if !seen.insert(*domain) {
2347 return Err(invalid_admission(
2348 "capacity update contains a duplicate domain",
2349 ));
2350 }
2351 let domain_state = state.domains.get(domain).ok_or_else(|| {
2352 admission_fault(
2353 DynamicAdmissionFaultKind::UnknownDomain,
2354 "capacity update references unknown domain",
2355 )
2356 })?;
2357 if new_total.get() < domain_state.used
2358 || new_total.get() > domain_state.spec.maximum_total_units.get()
2359 {
2360 return Err(invalid_admission(
2361 "capacity update is below live use or above maximum total",
2362 ));
2363 }
2364 if new_total.get() > domain_state.spec.total_units.get()
2365 && domain_state.availability_epoch == u64::MAX
2366 {
2367 return Err(admission_fault(
2368 DynamicAdmissionFaultKind::EpochExhausted,
2369 "domain availability epoch is exhausted",
2370 ));
2371 }
2372 changed |= *new_total != domain_state.spec.total_units;
2373 }
2374 if !changed {
2375 return Ok(state.epochs(self.id()));
2376 }
2377 let next_capacity_epoch = state.capacity_epoch.checked_add(1).ok_or_else(|| {
2378 admission_fault(
2379 DynamicAdmissionFaultKind::EpochExhausted,
2380 "capacity epoch is exhausted",
2381 )
2382 })?;
2383 for (domain, new_total) in updates {
2384 let domain_state = state
2385 .domains
2386 .get_mut(domain)
2387 .expect("validated capacity domain remains registered");
2388 if new_total.get() > domain_state.spec.total_units.get() {
2389 domain_state.availability_epoch += 1;
2390 }
2391 domain_state.spec.total_units = *new_total;
2392 }
2393 state.capacity_epoch = next_capacity_epoch;
2394 let epochs = state.epochs(self.id());
2395 self.inner.epoch_tx.send_replace(epochs);
2396 drop(state);
2397 Ok(epochs)
2398 }
2399
2400 pub(crate) fn notify_domain_availability_changed(
2404 &self,
2405 domain: CapacityDomainId,
2406 ) -> Result<CapacityEpochs, VNextError> {
2407 let mut state = self.inner.lock_mutation()?;
2408 if state.poisoned {
2409 return Err(admission_fault(
2410 DynamicAdmissionFaultKind::Poisoned,
2411 "coordinator is fail-closed",
2412 ));
2413 }
2414 if !state.domains.contains_key(&domain) {
2415 return Err(admission_fault(
2416 DynamicAdmissionFaultKind::UnknownDomain,
2417 "availability update references unknown domain",
2418 ));
2419 }
2420 let next_capacity_epoch = state.capacity_epoch.checked_add(1).ok_or_else(|| {
2421 admission_fault(
2422 DynamicAdmissionFaultKind::EpochExhausted,
2423 "capacity epoch is exhausted",
2424 )
2425 })?;
2426 let domain_state = state
2427 .domains
2428 .get_mut(&domain)
2429 .expect("validated availability domain remains registered");
2430 domain_state.availability_epoch = domain_state
2431 .availability_epoch
2432 .checked_add(1)
2433 .ok_or_else(|| {
2434 admission_fault(
2435 DynamicAdmissionFaultKind::EpochExhausted,
2436 "domain availability epoch is exhausted",
2437 )
2438 })?;
2439 state.capacity_epoch = next_capacity_epoch;
2440 let epochs = state.epochs(self.id());
2441 self.inner.epoch_tx.send_replace(epochs);
2442 drop(state);
2443 Ok(epochs)
2444 }
2445
2446 pub fn register_waiter(
2447 &self,
2448 observed: CapacityWaitCondition,
2449 ) -> Result<CapacityWaitRegistration, VNextError> {
2450 if observed.coordinator_id != self.id() {
2451 return Err(admission_fault(
2452 DynamicAdmissionFaultKind::ForeignCoordinator,
2453 "wait observation belongs to a different coordinator",
2454 ));
2455 }
2456 let receiver = self.inner.epoch_tx.subscribe();
2457 let state = self.inner.lock_state()?;
2458 if state.poisoned {
2459 return Err(admission_fault(
2460 DynamicAdmissionFaultKind::Poisoned,
2461 "coordinator is fail-closed",
2462 ));
2463 }
2464 let registered = state.refresh_wait_condition(self.id(), &observed)?;
2465 Ok(CapacityWaitRegistration {
2466 inner: Arc::clone(&self.inner),
2467 observed,
2468 registered,
2469 receiver,
2470 })
2471 }
2472}
2473
2474fn shortfall(
2475 requested: CapacityEntry,
2476 state: &DomainState,
2477 available: u64,
2478 kind: CapacityShortfallKind,
2479) -> CapacityShortfall {
2480 CapacityShortfall {
2481 domain: Some(requested.domain),
2482 kind,
2483 requested: requested.units,
2484 available: CapacityUnits::new(available),
2485 current_total: state.spec.total_units,
2486 maximum_total: state.spec.maximum_total_units,
2487 }
2488}
2489
2490struct DemandEvaluation {
2491 permanent: Vec<CapacityShortfall>,
2492 blockers: Vec<CapacityShortfall>,
2493 growth_required: bool,
2494}
2495
2496fn evaluate_demand(
2497 state: &CoordinatorState,
2498 demand: &AdmissionDemand,
2499) -> Result<DemandEvaluation, VNextError> {
2500 let mut permanent = Vec::new();
2501 for requested in demand
2502 .immediate_claim
2503 .entries()
2504 .iter()
2505 .chain(demand.fit_requirement.entries())
2506 {
2507 let Some(domain) = state.domains.get(&requested.domain) else {
2508 return Err(admission_fault(
2509 DynamicAdmissionFaultKind::UnknownDomain,
2510 format!(
2511 "demand references unknown capacity domain {}",
2512 requested.domain.get()
2513 ),
2514 ));
2515 };
2516 if requested.units.get() > domain.spec.maximum_total_units.get() {
2517 permanent.push(CapacityShortfall {
2518 domain: Some(requested.domain),
2519 kind: CapacityShortfallKind::PermanentDomainMaximum,
2520 requested: requested.units,
2521 available: CapacityUnits::new(
2522 domain.spec.total_units.get().saturating_sub(domain.used),
2523 ),
2524 current_total: domain.spec.total_units,
2525 maximum_total: domain.spec.maximum_total_units,
2526 });
2527 }
2528 }
2529 permanent.sort_by_key(|shortfall| (shortfall.domain, shortfall.kind as u8));
2530 permanent.dedup_by(|left, right| {
2531 left.domain == right.domain && left.kind == right.kind && left.requested == right.requested
2532 });
2533
2534 let mut blockers = Vec::new();
2535 let mut growth_required = false;
2536 if permanent.is_empty() {
2537 for requested in demand.immediate_claim.entries() {
2538 let domain = state
2539 .domains
2540 .get(&requested.domain)
2541 .expect("known demand domain was preflight validated");
2542 let available = domain.spec.total_units.get().saturating_sub(domain.used);
2543 if requested.units.get() > domain.spec.total_units.get() {
2544 growth_required = true;
2545 blockers.push(shortfall(
2546 *requested,
2547 domain,
2548 available,
2549 CapacityShortfallKind::BackingGrowthRequired,
2550 ));
2551 } else if requested.units.get() > available {
2552 blockers.push(shortfall(
2553 *requested,
2554 domain,
2555 available,
2556 CapacityShortfallKind::ImmediateAvailability,
2557 ));
2558 }
2559 }
2560 if demand.fit_policy == AdmissionFitPolicy::FullInputMustFit {
2561 for requested in demand.fit_requirement.entries() {
2562 let domain = state
2563 .domains
2564 .get(&requested.domain)
2565 .expect("known fit domain was preflight validated");
2566 let available = domain.spec.total_units.get().saturating_sub(domain.used);
2567 if requested.units.get() > domain.spec.total_units.get() {
2568 growth_required = true;
2569 blockers.push(shortfall(
2570 *requested,
2571 domain,
2572 available,
2573 CapacityShortfallKind::BackingGrowthRequired,
2574 ));
2575 } else if requested.units.get() > available {
2576 blockers.push(shortfall(
2577 *requested,
2578 domain,
2579 available,
2580 CapacityShortfallKind::FitAvailability,
2581 ));
2582 }
2583 }
2584 }
2585 }
2586 Ok(DemandEvaluation {
2587 permanent,
2588 blockers,
2589 growth_required,
2590 })
2591}
2592
2593fn deferred_action(demand: &AdmissionDemand, growth_required: bool) -> DeferredAction {
2594 if growth_required {
2595 DeferredAction::AwaitBackingGrowth
2596 } else {
2597 match demand.pressure_action {
2598 AdmissionPressureAction::WaitForRelease => DeferredAction::WaitForRelease,
2599 AdmissionPressureAction::PreemptAndRecompute => DeferredAction::PreemptAndRecompute,
2600 }
2601 }
2602}
2603
2604#[derive(Debug)]
2605#[must_use = "dropping the request lease releases request-scoped capacity"]
2606pub struct LogicalRequestLease {
2607 inner: Arc<CoordinatorInner>,
2608 request: RequestAuthorityId,
2609 claims: CapacityVector,
2610 released: bool,
2611}
2612
2613impl LogicalRequestLease {
2614 pub fn coordinator_id(&self) -> LogicalAdmissionCoordinatorId {
2615 self.inner.id
2616 }
2617
2618 pub const fn request(&self) -> RequestAuthorityId {
2619 self.request
2620 }
2621
2622 pub fn claims(&self) -> &CapacityVector {
2623 &self.claims
2624 }
2625
2626 fn release_inner(&mut self) -> bool {
2627 if self.released {
2628 return true;
2629 }
2630 let state = match self.inner.state.lock() {
2631 Ok(state) => state,
2632 Err(poisoned) => {
2633 let mut state = poisoned.into_inner();
2634 state.poisoned = true;
2635 let epochs = state.epochs(self.coordinator_id());
2636 self.inner.epoch_tx.send_replace(epochs);
2637 return false;
2638 }
2639 };
2640 let mut state = CoordinatorMutationGuard {
2641 inner: self.inner.as_ref(),
2642 state,
2643 panicking_on_entry: std::thread::panicking(),
2644 };
2645 let request_is_releasable = state
2646 .live_requests
2647 .get(self.request.sparse_id as usize)
2648 .and_then(|record| *record)
2649 .is_some_and(|record| {
2650 record.generation == self.request.generation && record.active_sequences == 0
2651 });
2652 if state.poisoned
2653 || !request_is_releasable
2654 || state.active_requests == 0
2655 || self.claims.entries().iter().any(|claim| {
2656 state
2657 .domains
2658 .get(&claim.domain)
2659 .is_none_or(|domain| domain.used < claim.units.get())
2660 })
2661 {
2662 state.poisoned = true;
2663 let epochs = state.epochs(self.coordinator_id());
2664 self.inner.epoch_tx.send_replace(epochs);
2665 return false;
2666 }
2667 let Some(next_release_epoch) = state.release_epoch.checked_add(1) else {
2668 state.poisoned = true;
2669 let epochs = state.epochs(self.coordinator_id());
2670 self.inner.epoch_tx.send_replace(epochs);
2671 return false;
2672 };
2673 if self.claims.entries().iter().any(|claim| {
2674 state
2675 .domains
2676 .get(&claim.domain)
2677 .is_some_and(|domain| domain.availability_epoch == u64::MAX)
2678 }) {
2679 state.poisoned = true;
2680 let epochs = state.epochs(self.coordinator_id());
2681 self.inner.epoch_tx.send_replace(epochs);
2682 return false;
2683 }
2684 for claim in self.claims.entries() {
2685 let domain = state
2686 .domains
2687 .get_mut(&claim.domain)
2688 .expect("validated request capacity domain remains registered");
2689 domain.used -= claim.units.get();
2690 domain.availability_epoch += 1;
2691 }
2692 state.active_requests -= 1;
2693 state.live_requests[self.request.sparse_id as usize] = None;
2694 state.reusable_request_ids.push(self.request.sparse_id);
2695 state.release_epoch = next_release_epoch;
2696 let epochs = state.epochs(self.coordinator_id());
2697 self.inner.epoch_tx.send_replace(epochs);
2698 drop(state);
2699 self.released = true;
2700 true
2701 }
2702}
2703
2704impl Drop for LogicalRequestLease {
2705 fn drop(&mut self) {
2706 let _ = self.release_inner();
2707 }
2708}
2709
2710#[derive(Debug)]
2711#[must_use = "dropping the logical admission lease releases its capacity claim"]
2712pub struct LogicalAdmissionLease {
2713 inner: Arc<CoordinatorInner>,
2714 request: RequestAuthorityId,
2715 sequence: SequenceAuthorityId,
2716 claims: CapacityVector,
2717 released: bool,
2718}
2719
2720impl LogicalAdmissionLease {
2721 pub fn coordinator_id(&self) -> LogicalAdmissionCoordinatorId {
2722 self.inner.id
2723 }
2724
2725 pub const fn sequence(&self) -> SequenceAuthorityId {
2726 self.sequence
2727 }
2728
2729 pub const fn request(&self) -> RequestAuthorityId {
2730 self.request
2731 }
2732
2733 pub fn claims(&self) -> &CapacityVector {
2734 &self.claims
2735 }
2736
2737 fn release_inner(&mut self) -> bool {
2738 if self.released {
2739 return true;
2740 }
2741 let state = match self.inner.state.lock() {
2742 Ok(state) => state,
2743 Err(poisoned) => {
2744 let mut state = poisoned.into_inner();
2745 state.poisoned = true;
2746 let epochs = state.epochs(self.coordinator_id());
2747 self.inner.epoch_tx.send_replace(epochs);
2748 return false;
2749 }
2750 };
2751 let mut state = CoordinatorMutationGuard {
2752 inner: self.inner.as_ref(),
2753 state,
2754 panicking_on_entry: std::thread::panicking(),
2755 };
2756 let sequence_record = state
2757 .live_sequences
2758 .get(self.sequence.sparse_id as usize)
2759 .and_then(|record| *record);
2760 let request_record = state
2761 .live_requests
2762 .get(self.request.sparse_id as usize)
2763 .and_then(|record| *record);
2764 if state.poisoned
2765 || sequence_record.is_none_or(|record| {
2766 record.generation != self.sequence.generation
2767 || record.request != self.request
2768 || record.active_child_claims != 0
2769 })
2770 || request_record.is_none_or(|record| {
2771 record.generation != self.request.generation || record.active_sequences == 0
2772 })
2773 || state.active_sequences == 0
2774 || self.claims.entries().iter().any(|claim| {
2775 state
2776 .domains
2777 .get(&claim.domain)
2778 .is_none_or(|domain| domain.used < claim.units.get())
2779 })
2780 {
2781 state.poisoned = true;
2782 let epochs = state.epochs(self.coordinator_id());
2783 self.inner.epoch_tx.send_replace(epochs);
2784 return false;
2785 }
2786 let Some(next_release_epoch) = state.release_epoch.checked_add(1) else {
2787 state.poisoned = true;
2788 let epochs = state.epochs(self.coordinator_id());
2789 self.inner.epoch_tx.send_replace(epochs);
2790 return false;
2791 };
2792 if state.active_sequence_availability_epoch == u64::MAX
2793 || self.claims.entries().iter().any(|claim| {
2794 state
2795 .domains
2796 .get(&claim.domain)
2797 .is_some_and(|domain| domain.availability_epoch == u64::MAX)
2798 })
2799 {
2800 state.poisoned = true;
2801 let epochs = state.epochs(self.coordinator_id());
2802 self.inner.epoch_tx.send_replace(epochs);
2803 return false;
2804 }
2805 for claim in self.claims.entries() {
2806 let domain = state
2807 .domains
2808 .get_mut(&claim.domain)
2809 .expect("validated logical claim domain remains registered");
2810 domain.used -= claim.units.get();
2811 domain.availability_epoch += 1;
2812 }
2813 state.active_sequences -= 1;
2814 state.active_sequence_availability_epoch += 1;
2815 state.live_requests[self.request.sparse_id as usize]
2816 .as_mut()
2817 .expect("validated parent request remains live")
2818 .active_sequences -= 1;
2819 state.live_sequences[self.sequence.sparse_id as usize] = None;
2820 state.reusable_sequence_ids.push(self.sequence.sparse_id);
2821 state.release_epoch = next_release_epoch;
2822 let epochs = state.epochs(self.coordinator_id());
2823 self.inner.epoch_tx.send_replace(epochs);
2824 drop(state);
2825 self.released = true;
2826 true
2827 }
2828}
2829
2830impl Drop for LogicalAdmissionLease {
2831 fn drop(&mut self) {
2832 let _ = self.release_inner();
2833 }
2834}
2835
2836#[derive(Debug)]
2837#[must_use = "dropping a child capacity lease releases its exact domain claims"]
2838pub struct LogicalCapacityLease {
2839 batch: LogicalBatchCapacityLease,
2840}
2841
2842impl LogicalCapacityLease {
2843 pub fn coordinator_id(&self) -> LogicalAdmissionCoordinatorId {
2844 self.batch.coordinator_id()
2845 }
2846
2847 pub fn sequence(&self) -> SequenceAuthorityId {
2848 self.batch.parents[0].sequence
2849 }
2850
2851 pub fn request(&self) -> RequestAuthorityId {
2852 self.batch.parents[0].request
2853 }
2854
2855 pub fn claims(&self) -> &CapacityVector {
2856 self.batch.claims()
2857 }
2858}
2859
2860#[derive(Debug)]
2861#[must_use = "dropping a batch child lease releases its exact shared claim"]
2862pub struct LogicalBatchCapacityLease {
2863 inner: Arc<CoordinatorInner>,
2864 parents: Vec<SequenceCapacityParent>,
2865 claims: CapacityVector,
2866 released: bool,
2867}
2868
2869impl LogicalBatchCapacityLease {
2870 pub fn coordinator_id(&self) -> LogicalAdmissionCoordinatorId {
2871 self.inner.id
2872 }
2873
2874 pub fn parents(&self) -> &[SequenceCapacityParent] {
2875 &self.parents
2876 }
2877
2878 pub fn claims(&self) -> &CapacityVector {
2879 &self.claims
2880 }
2881
2882 fn release_inner(&mut self) -> bool {
2883 if self.released {
2884 return true;
2885 }
2886 let state = match self.inner.state.lock() {
2887 Ok(state) => state,
2888 Err(poisoned) => {
2889 let mut state = poisoned.into_inner();
2890 state.poisoned = true;
2891 let epochs = state.epochs(self.coordinator_id());
2892 self.inner.epoch_tx.send_replace(epochs);
2893 return false;
2894 }
2895 };
2896 let mut state = CoordinatorMutationGuard {
2897 inner: self.inner.as_ref(),
2898 state,
2899 panicking_on_entry: std::thread::panicking(),
2900 };
2901 let parents_are_live = !self.parents.is_empty()
2902 && self.parents.iter().all(|parent| {
2903 state
2904 .live_sequences
2905 .get(parent.sequence.sparse_id as usize)
2906 .and_then(|record| *record)
2907 .is_some_and(|record| {
2908 record.generation == parent.sequence.generation
2909 && record.request == parent.request
2910 && record.active_child_claims > 0
2911 })
2912 });
2913 if state.poisoned
2914 || !parents_are_live
2915 || state.active_child_claims == 0
2916 || self.claims.entries().iter().any(|claim| {
2917 state
2918 .domains
2919 .get(&claim.domain)
2920 .is_none_or(|domain| domain.used < claim.units.get())
2921 })
2922 {
2923 state.poisoned = true;
2924 let epochs = state.epochs(self.coordinator_id());
2925 self.inner.epoch_tx.send_replace(epochs);
2926 return false;
2927 }
2928 let Some(next_release_epoch) = state.release_epoch.checked_add(1) else {
2929 state.poisoned = true;
2930 let epochs = state.epochs(self.coordinator_id());
2931 self.inner.epoch_tx.send_replace(epochs);
2932 return false;
2933 };
2934 if self.claims.entries().iter().any(|claim| {
2935 state
2936 .domains
2937 .get(&claim.domain)
2938 .is_some_and(|domain| domain.availability_epoch == u64::MAX)
2939 }) {
2940 state.poisoned = true;
2941 let epochs = state.epochs(self.coordinator_id());
2942 self.inner.epoch_tx.send_replace(epochs);
2943 return false;
2944 }
2945 for claim in self.claims.entries() {
2946 let domain = state
2947 .domains
2948 .get_mut(&claim.domain)
2949 .expect("validated child capacity domain remains registered");
2950 domain.used -= claim.units.get();
2951 domain.availability_epoch += 1;
2952 }
2953 state.active_child_claims -= 1;
2954 for parent in &self.parents {
2955 state.live_sequences[parent.sequence.sparse_id as usize]
2956 .as_mut()
2957 .expect("validated batch parent sequence remains live")
2958 .active_child_claims -= 1;
2959 }
2960 state.release_epoch = next_release_epoch;
2961 let epochs = state.epochs(self.coordinator_id());
2962 self.inner.epoch_tx.send_replace(epochs);
2963 drop(state);
2964 self.released = true;
2965 true
2966 }
2967}
2968
2969impl Drop for LogicalBatchCapacityLease {
2970 fn drop(&mut self) {
2971 let _ = self.release_inner();
2972 }
2973}
2974
2975#[derive(Debug)]
2976pub struct CapacityWaitRegistration {
2977 inner: Arc<CoordinatorInner>,
2978 observed: CapacityWaitCondition,
2979 registered: CapacityWaitCondition,
2980 receiver: watch::Receiver<CapacityEpochs>,
2981}
2982
2983#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2984pub struct CapacityWaitRecheck {
2985 current: CapacityEpochs,
2986 changed_since_observation: bool,
2987 changed_since_registration: bool,
2988}
2989
2990impl CapacityWaitRegistration {
2991 pub fn recheck(&self) -> Result<CapacityWaitRecheck, VNextError> {
2992 let state = self.inner.lock_state()?;
2993 if state.poisoned {
2994 return Err(admission_fault(
2995 DynamicAdmissionFaultKind::Poisoned,
2996 "coordinator is fail-closed",
2997 ));
2998 }
2999 let current = state.epochs(self.inner.id);
3000 Ok(CapacityWaitRecheck {
3001 current,
3002 changed_since_observation: state
3003 .wait_condition_changed(self.inner.id, &self.observed)?,
3004 changed_since_registration: state
3005 .wait_condition_changed(self.inner.id, &self.registered)?,
3006 })
3007 }
3008
3009 pub async fn wait_for_change(mut self) -> Result<CapacityEpochs, VNextError> {
3010 loop {
3011 let recheck = self.recheck()?;
3012 if recheck.should_retry() {
3013 return Ok(recheck.current());
3014 }
3015 self.receiver.changed().await.map_err(|_| {
3016 admission_fault(
3017 DynamicAdmissionFaultKind::Poisoned,
3018 "capacity epoch listener closed",
3019 )
3020 })?;
3021 self.receiver.borrow_and_update();
3022 }
3023 }
3024}
3025
3026impl CapacityWaitRecheck {
3027 pub(crate) const fn new(
3028 current: CapacityEpochs,
3029 changed_since_observation: bool,
3030 changed_since_registration: bool,
3031 ) -> Self {
3032 Self {
3033 current,
3034 changed_since_observation,
3035 changed_since_registration,
3036 }
3037 }
3038
3039 pub const fn current(self) -> CapacityEpochs {
3040 self.current
3041 }
3042
3043 pub const fn changed_since_observation(self) -> bool {
3044 self.changed_since_observation
3045 }
3046
3047 pub const fn changed_since_registration(self) -> bool {
3048 self.changed_since_registration
3049 }
3050
3051 pub const fn should_retry(self) -> bool {
3052 self.changed_since_observation || self.changed_since_registration
3053 }
3054}
3055
3056#[cfg(test)]
3057mod tests {
3058 use super::*;
3059 use std::sync::{Arc, Condvar, Mutex as StdMutex};
3060 use std::thread;
3061
3062 const TEST_NATIVE_WORKER_LIMIT: usize = 8;
3063
3064 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
3065 enum TestGatePhase {
3066 Holding,
3067 Released,
3068 Cancelled,
3069 }
3070
3071 #[derive(Debug)]
3072 struct TestGateState {
3073 ready: usize,
3074 phase: TestGatePhase,
3075 }
3076
3077 #[derive(Debug)]
3078 struct CancellableTestGate {
3079 state: StdMutex<TestGateState>,
3080 changed: Condvar,
3081 }
3082
3083 impl CancellableTestGate {
3084 fn new(worker_count: usize) -> Arc<Self> {
3085 assert!(worker_count > 0);
3086 assert!(worker_count <= TEST_NATIVE_WORKER_LIMIT);
3087 Arc::new(Self {
3088 state: StdMutex::new(TestGateState {
3089 ready: 0,
3090 phase: TestGatePhase::Holding,
3091 }),
3092 changed: Condvar::new(),
3093 })
3094 }
3095
3096 fn arrive_and_wait(&self) -> bool {
3097 let mut state = self.state.lock().unwrap();
3098 if state.phase != TestGatePhase::Holding {
3099 return state.phase == TestGatePhase::Released;
3100 }
3101 state.ready += 1;
3102 self.changed.notify_all();
3103 while state.phase == TestGatePhase::Holding {
3104 state = self.changed.wait(state).unwrap();
3105 }
3106 state.phase == TestGatePhase::Released
3107 }
3108
3109 fn wait_until_ready(&self, worker_count: usize) -> bool {
3110 assert!(worker_count <= TEST_NATIVE_WORKER_LIMIT);
3111 let mut state = self.state.lock().unwrap();
3112 while state.ready < worker_count && state.phase == TestGatePhase::Holding {
3113 state = self.changed.wait(state).unwrap();
3114 }
3115 state.ready == worker_count && state.phase == TestGatePhase::Holding
3116 }
3117
3118 fn release(&self) {
3119 let mut state = self.state.lock().unwrap();
3120 assert_eq!(state.phase, TestGatePhase::Holding);
3121 state.phase = TestGatePhase::Released;
3122 self.changed.notify_all();
3123 }
3124
3125 fn cancel(&self) {
3126 let mut state = self
3127 .state
3128 .lock()
3129 .unwrap_or_else(|poisoned| poisoned.into_inner());
3130 if state.phase == TestGatePhase::Holding {
3131 state.phase = TestGatePhase::Cancelled;
3132 self.changed.notify_all();
3133 }
3134 }
3135 }
3136
3137 struct CancelTestGateOnDrop<'a> {
3138 gate: &'a CancellableTestGate,
3139 armed: bool,
3140 }
3141
3142 impl<'a> CancelTestGateOnDrop<'a> {
3143 fn new(gate: &'a CancellableTestGate) -> Self {
3144 Self { gate, armed: true }
3145 }
3146
3147 fn disarm(&mut self) {
3148 self.armed = false;
3149 }
3150 }
3151
3152 impl Drop for CancelTestGateOnDrop<'_> {
3153 fn drop(&mut self) {
3154 if self.armed {
3155 self.gate.cancel();
3156 }
3157 }
3158 }
3159
3160 fn cancel_gate_on_unwind<T>(gate: &CancellableTestGate, work: impl FnOnce() -> T) -> T {
3161 match std::panic::catch_unwind(std::panic::AssertUnwindSafe(work)) {
3162 Ok(value) => value,
3163 Err(payload) => {
3164 gate.cancel();
3165 std::panic::resume_unwind(payload);
3166 }
3167 }
3168 }
3169
3170 fn domain(value: u32) -> CapacityDomainId {
3171 CapacityDomainId::new(value).unwrap()
3172 }
3173
3174 fn vector(entries: &[(u32, u64)]) -> CapacityVector {
3175 CapacityVector::new(
3176 entries
3177 .iter()
3178 .map(|(domain_id, units)| {
3179 CapacityEntry::new(domain(*domain_id), CapacityUnits::new(*units)).unwrap()
3180 })
3181 .collect(),
3182 )
3183 .unwrap()
3184 }
3185
3186 fn demand(immediate: &[(u32, u64)], fit: &[(u32, u64)]) -> AdmissionDemand {
3187 AdmissionDemand::from_plan(
3188 vector(immediate),
3189 vector(fit),
3190 AdmissionFitPolicy::FullInputMustFit,
3191 AdmissionPressureAction::WaitForRelease,
3192 )
3193 .unwrap()
3194 }
3195
3196 fn coordinator(maximum_active_sequences: u32) -> LogicalAdmissionCoordinator {
3197 LogicalAdmissionCoordinator::new(
3198 vec![
3199 (
3200 domain(1),
3201 CapacityDomainSpec::new(CapacityUnits::new(10), CapacityUnits::new(20))
3202 .unwrap(),
3203 ),
3204 (
3205 domain(2),
3206 CapacityDomainSpec::new(CapacityUnits::new(4), CapacityUnits::new(4)).unwrap(),
3207 ),
3208 ],
3209 maximum_active_sequences,
3210 )
3211 .unwrap()
3212 }
3213
3214 fn wait_for_domains(
3215 coordinator: &LogicalAdmissionCoordinator,
3216 domains: &[u32],
3217 ) -> CapacityWaitCondition {
3218 coordinator
3219 .wait_snapshot_for_domains(domains.iter().copied().map(domain))
3220 .unwrap()
3221 .wait_condition()
3222 .clone()
3223 }
3224
3225 fn admitted(decision: AdmissionDecision) -> LogicalAdmissionLease {
3226 match decision {
3227 AdmissionDecision::Admitted(lease) => lease,
3228 _ => panic!("expected admitted decision"),
3229 }
3230 }
3231
3232 fn empty_demand() -> AdmissionDemand {
3233 AdmissionDemand::from_plan(
3234 CapacityVector::empty(),
3235 CapacityVector::empty(),
3236 AdmissionFitPolicy::ImmediateOnly,
3237 AdmissionPressureAction::WaitForRelease,
3238 )
3239 .unwrap()
3240 }
3241
3242 fn admitted_request(decision: RequestAdmissionDecision) -> LogicalRequestLease {
3243 match decision {
3244 RequestAdmissionDecision::Admitted(lease) => lease,
3245 _ => panic!("expected admitted request"),
3246 }
3247 }
3248
3249 fn request(coordinator: &LogicalAdmissionCoordinator) -> LogicalRequestLease {
3250 admitted_request(coordinator.try_admit_request(&empty_demand()).unwrap())
3251 }
3252
3253 fn admit_sequence(
3254 coordinator: &LogicalAdmissionCoordinator,
3255 request: &LogicalRequestLease,
3256 demand: &AdmissionDemand,
3257 ) -> LogicalAdmissionLease {
3258 admitted(
3259 coordinator
3260 .try_admit_sequence_for_request(request, demand)
3261 .unwrap(),
3262 )
3263 }
3264
3265 fn claimed_child(decision: CapacityClaimDecision) -> LogicalCapacityLease {
3266 match decision {
3267 CapacityClaimDecision::Claimed(lease) => lease,
3268 _ => panic!("expected child capacity claim"),
3269 }
3270 }
3271
3272 fn claimed_batch(decision: BatchCapacityClaimDecision) -> LogicalBatchCapacityLease {
3273 match decision {
3274 BatchCapacityClaimDecision::Claimed(lease) => lease,
3275 _ => panic!("expected batch child capacity claim"),
3276 }
3277 }
3278
3279 #[test]
3280 fn request_capacity_is_shared_once_across_multiple_sequences() {
3281 let coordinator = coordinator(2);
3282 let request = admitted_request(
3283 coordinator
3284 .try_admit_request(&demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]))
3285 .unwrap(),
3286 );
3287 let first = admit_sequence(
3288 &coordinator,
3289 &request,
3290 &demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]),
3291 );
3292 let second = admit_sequence(
3293 &coordinator,
3294 &request,
3295 &demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]),
3296 );
3297 let both = coordinator.snapshot().unwrap();
3298 assert_eq!(both.active_requests(), 1);
3299 assert_eq!(both.active_sequences(), 2);
3300 assert_eq!(both.domains()[0].used().get(), 6);
3301 assert_eq!(both.domains()[1].used().get(), 3);
3302 assert_eq!(first.request(), request.request());
3303 assert_eq!(second.request(), request.request());
3304
3305 drop(first);
3306 let one = coordinator.snapshot().unwrap();
3307 assert_eq!(one.active_requests(), 1);
3308 assert_eq!(one.active_sequences(), 1);
3309 assert_eq!(one.domains()[0].used().get(), 4);
3310 assert_eq!(one.domains()[1].used().get(), 2);
3311
3312 drop(second);
3313 let request_only = coordinator.snapshot().unwrap();
3314 assert_eq!(request_only.active_requests(), 1);
3315 assert_eq!(request_only.active_sequences(), 0);
3316 assert_eq!(request_only.domains()[0].used().get(), 2);
3317 assert_eq!(request_only.domains()[1].used().get(), 1);
3318 drop(request);
3319 let empty = coordinator.snapshot().unwrap();
3320 assert_eq!(empty.active_requests(), 0);
3321 assert!(empty
3322 .domains()
3323 .iter()
3324 .all(|domain| domain.used().get() == 0));
3325 }
3326
3327 #[test]
3328 fn request_defer_and_reject_are_atomic_and_do_not_consume_sequence_slots() {
3329 let coordinator = coordinator(2);
3330 let held = admitted_request(
3331 coordinator
3332 .try_admit_request(&demand(&[(1, 1), (2, 3)], &[(1, 1), (2, 3)]))
3333 .unwrap(),
3334 );
3335 let before = coordinator.snapshot().unwrap();
3336 let deferred = coordinator
3337 .try_admit_request(&demand(&[(1, 1), (2, 2)], &[(1, 1), (2, 2)]))
3338 .unwrap();
3339 assert!(matches!(deferred, RequestAdmissionDecision::Deferred(_)));
3340 let rejected = coordinator
3341 .try_admit_request(&demand(&[(1, 21), (2, 1)], &[(1, 21), (2, 1)]))
3342 .unwrap();
3343 assert!(matches!(
3344 rejected,
3345 RequestAdmissionDecision::PermanentRejected(_)
3346 ));
3347 let after = coordinator.snapshot().unwrap();
3348 assert_eq!(before.domains, after.domains);
3349 assert_eq!(after.active_requests(), 1);
3350 assert_eq!(after.active_sequences(), 0);
3351 drop(held);
3352 }
3353
3354 #[test]
3355 fn initial_sequence_defer_retains_no_partial_request_or_capacity() {
3356 let coordinator = coordinator(2);
3357 let held = match coordinator
3358 .try_admit_initial_sequence(
3359 &demand(&[(1, 1)], &[(1, 1)]),
3360 &demand(&[(2, 3)], &[(2, 3)]),
3361 )
3362 .unwrap()
3363 {
3364 InitialSequenceAdmissionDecision::Admitted(bundle) => bundle,
3365 _ => panic!("first initial bundle must be admitted"),
3366 };
3367 let before = coordinator.snapshot().unwrap();
3368
3369 assert!(matches!(
3370 coordinator
3371 .try_admit_initial_sequence(
3372 &demand(&[(1, 1)], &[(1, 1)]),
3373 &demand(&[(2, 2)], &[(2, 2)]),
3374 )
3375 .unwrap(),
3376 InitialSequenceAdmissionDecision::Deferred
3377 ));
3378 let after = coordinator.snapshot().unwrap();
3379 assert_eq!(after.active_requests(), before.active_requests());
3380 assert_eq!(after.active_sequences(), before.active_sequences());
3381 assert_eq!(after.domains, before.domains);
3382
3383 drop(held);
3384 let empty = coordinator.snapshot().unwrap();
3385 assert_eq!(empty.active_requests(), 0);
3386 assert_eq!(empty.active_sequences(), 0);
3387 assert!(empty
3388 .domains()
3389 .iter()
3390 .all(|domain| domain.used().get() == 0));
3391 }
3392
3393 #[test]
3394 fn initial_sequence_bundle_sums_overlapping_domains_and_releases_child_first() {
3395 let coordinator = coordinator(1);
3396 let bundle = match coordinator
3397 .try_admit_initial_sequence(
3398 &demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]),
3399 &demand(&[(1, 3), (2, 2)], &[(1, 3), (2, 2)]),
3400 )
3401 .unwrap()
3402 {
3403 InitialSequenceAdmissionDecision::Admitted(bundle) => bundle,
3404 _ => panic!("overlapping initial bundle must be admitted atomically"),
3405 };
3406 let admitted = coordinator.snapshot().unwrap();
3407 assert_eq!(admitted.active_requests(), 1);
3408 assert_eq!(admitted.active_sequences(), 1);
3409 assert_eq!(admitted.domains()[0].used().get(), 5);
3410 assert_eq!(admitted.domains()[1].used().get(), 3);
3411
3412 let (request, sequence) = bundle.into_parts();
3413 assert_eq!(request.request(), sequence.request());
3414 drop(sequence);
3415 drop(request);
3416 let released = coordinator.snapshot().unwrap();
3417 assert!(!released.poisoned());
3418 assert_eq!(released.active_requests(), 0);
3419 assert_eq!(released.active_sequences(), 0);
3420 assert!(released
3421 .domains()
3422 .iter()
3423 .all(|domain| domain.used().get() == 0));
3424 }
3425
3426 #[test]
3427 fn request_authority_reuses_sparse_storage_with_new_generation() {
3428 let coordinator = coordinator(u32::MAX);
3429 let mut previous_generation = 0;
3430 for _ in 0..100_000 {
3431 let request = request(&coordinator);
3432 assert_eq!(request.request().sparse_id(), 0);
3433 assert!(request.request().generation() > previous_generation);
3434 previous_generation = request.request().generation();
3435 drop(request);
3436 }
3437 let snapshot = coordinator.snapshot().unwrap();
3438 assert_eq!(snapshot.active_requests(), 0);
3439 assert_eq!(snapshot.live_request_records(), 0);
3440 assert_eq!(snapshot.reusable_request_ids(), 1);
3441 assert_eq!(snapshot.release_epoch(), 100_001);
3442 }
3443
3444 #[test]
3445 fn early_request_release_with_live_sequence_fails_closed_and_retains_claims() {
3446 let coordinator = coordinator(1);
3447 let mut request = admitted_request(
3448 coordinator
3449 .try_admit_request(&demand(&[(1, 2)], &[(1, 2)]))
3450 .unwrap(),
3451 );
3452 let sequence = admit_sequence(&coordinator, &request, &demand(&[(1, 1)], &[(1, 1)]));
3453 assert!(!request.release_inner());
3454 let snapshot = coordinator.snapshot().unwrap();
3455 assert!(snapshot.poisoned());
3456 assert_eq!(snapshot.active_requests(), 1);
3457 assert_eq!(snapshot.active_sequences(), 1);
3458 assert_eq!(snapshot.domains()[0].used().get(), 3);
3459 drop(sequence);
3460 drop(request);
3461 }
3462
3463 #[test]
3464 fn exact_fit_claims_only_immediate_and_release_retries() {
3465 let coordinator = coordinator(2);
3466 let request = request(&coordinator);
3467 let first = admit_sequence(
3468 &coordinator,
3469 &request,
3470 &demand(&[(1, 6), (2, 2)], &[(1, 10), (2, 2)]),
3471 );
3472 let snapshot = coordinator.snapshot().unwrap();
3473 assert_eq!(snapshot.domains()[0].used().get(), 6);
3474 assert_eq!(snapshot.domains()[0].available().get(), 4);
3475
3476 let deferred = coordinator
3477 .try_admit_sequence_for_request(&request, &demand(&[(1, 5), (2, 1)], &[(1, 5), (2, 1)]))
3478 .unwrap();
3479 let observed = match deferred {
3480 AdmissionDecision::Deferred(value) => {
3481 assert_eq!(value.action(), DeferredAction::WaitForRelease);
3482 value.wait_condition().clone()
3483 }
3484 _ => panic!("expected capacity defer"),
3485 };
3486 drop(first);
3487 let registration = coordinator.register_waiter(observed).unwrap();
3488 assert!(registration.recheck().unwrap().should_retry());
3489 let retry = coordinator
3490 .try_admit_sequence_for_request(&request, &demand(&[(1, 5), (2, 1)], &[(1, 5), (2, 1)]))
3491 .unwrap();
3492 assert!(matches!(retry, AdmissionDecision::Admitted(_)));
3493 }
3494
3495 #[test]
3496 fn child_claim_uses_parent_authority_without_consuming_sequence_slot() {
3497 let coordinator = coordinator(1);
3498 let request = request(&coordinator);
3499 let parent = admit_sequence(
3500 &coordinator,
3501 &request,
3502 &demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]),
3503 );
3504 let before = coordinator.snapshot().unwrap();
3505 let child = claimed_child(
3506 coordinator
3507 .try_claim_for_sequence(&parent, &demand(&[(1, 3), (2, 1)], &[(1, 3), (2, 1)]))
3508 .unwrap(),
3509 );
3510 assert_eq!(child.sequence(), parent.sequence());
3511 assert!(coordinator.owns_capacity_claim(&child));
3512 let claimed = coordinator.snapshot().unwrap();
3513 assert_eq!(claimed.active_sequences(), 1);
3514 assert_eq!(claimed.active_child_claims(), 1);
3515 assert_eq!(claimed.domains()[0].used().get(), 5);
3516 assert_eq!(claimed.domains()[1].used().get(), 2);
3517
3518 drop(child);
3519 let released = coordinator.snapshot().unwrap();
3520 assert_eq!(released.active_sequences(), 1);
3521 assert_eq!(released.active_child_claims(), 0);
3522 assert_eq!(released.domains()[0].used().get(), 2);
3523 assert_eq!(released.domains()[1].used().get(), 1);
3524 assert_eq!(released.release_epoch(), before.release_epoch() + 1);
3525 drop(parent);
3526 assert_eq!(coordinator.snapshot().unwrap().active_sequences(), 0);
3527 }
3528
3529 #[test]
3530 fn batch_child_claim_charges_once_and_binds_every_parent() {
3531 let coordinator = coordinator(2);
3532 let first_request = request(&coordinator);
3533 let second_request = request(&coordinator);
3534 let first = admit_sequence(&coordinator, &first_request, &empty_demand());
3535 let second = admit_sequence(&coordinator, &second_request, &empty_demand());
3536 let before = coordinator.snapshot().unwrap();
3537
3538 let batch = claimed_batch(
3539 coordinator
3540 .try_claim_for_sequences(
3541 &[&second, &first],
3542 &demand(&[(1, 3), (2, 1)], &[(1, 3), (2, 1)]),
3543 )
3544 .unwrap(),
3545 );
3546 assert!(coordinator.owns_batch_capacity_claim(&batch));
3547 assert_eq!(batch.parents().len(), 2);
3548 assert!(batch
3549 .parents()
3550 .windows(2)
3551 .all(|pair| pair[0].sequence() < pair[1].sequence()));
3552 let claimed = coordinator.snapshot().unwrap();
3553 assert_eq!(claimed.active_requests(), 2);
3554 assert_eq!(claimed.active_sequences(), 2);
3555 assert_eq!(claimed.active_child_claims(), 1);
3556 assert_eq!(claimed.domains()[0].used().get(), 3);
3557 assert_eq!(claimed.domains()[1].used().get(), 1);
3558 let state = coordinator.inner.state.lock().unwrap();
3559 assert_eq!(
3560 state.live_sequences[first.sequence().sparse_id() as usize]
3561 .unwrap()
3562 .active_child_claims,
3563 1
3564 );
3565 assert_eq!(
3566 state.live_sequences[second.sequence().sparse_id() as usize]
3567 .unwrap()
3568 .active_child_claims,
3569 1
3570 );
3571 drop(state);
3572
3573 drop(batch);
3574 let released = coordinator.snapshot().unwrap();
3575 assert_eq!(released.active_child_claims(), 0);
3576 assert_eq!(released.domains()[0].used().get(), 0);
3577 assert_eq!(released.domains()[1].used().get(), 0);
3578 assert_eq!(released.release_epoch(), before.release_epoch() + 1);
3579 drop(first);
3580 drop(second);
3581 drop(first_request);
3582 drop(second_request);
3583 }
3584
3585 #[test]
3586 fn batch_child_rejects_duplicate_and_foreign_parents_atomically() {
3587 let local = coordinator(2);
3588 let local_request = request(&local);
3589 let first = admit_sequence(&local, &local_request, &empty_demand());
3590 let before = local.snapshot().unwrap();
3591 assert!(local
3592 .try_claim_for_sequences(&[&first, &first], &demand(&[(1, 1)], &[(1, 1)]))
3593 .is_err());
3594 let after_duplicate = local.snapshot().unwrap();
3595 assert_eq!(before.domains, after_duplicate.domains);
3596 assert_eq!(after_duplicate.active_child_claims(), 0);
3597
3598 let foreign = coordinator(1);
3599 let foreign_request = request(&foreign);
3600 let foreign_sequence = admit_sequence(&foreign, &foreign_request, &empty_demand());
3601 assert!(matches!(
3602 local.try_claim_for_sequences(
3603 &[&first, &foreign_sequence],
3604 &demand(&[(1, 1)], &[(1, 1)])
3605 ),
3606 Err(VNextError::DynamicAdmissionContract {
3607 kind: DynamicAdmissionFaultKind::ForeignCoordinator,
3608 ..
3609 })
3610 ));
3611 let after_foreign = local.snapshot().unwrap();
3612 assert_eq!(before.domains, after_foreign.domains);
3613 assert_eq!(after_foreign.active_child_claims(), 0);
3614 }
3615
3616 #[test]
3617 fn batch_child_defer_and_reject_have_zero_partial_parent_effect() {
3618 let coordinator = coordinator(2);
3619 let request = request(&coordinator);
3620 let first = admit_sequence(&coordinator, &request, &demand(&[(2, 3)], &[(2, 3)]));
3621 let second = admit_sequence(&coordinator, &request, &empty_demand());
3622 let before = coordinator.snapshot().unwrap();
3623
3624 let deferred = coordinator
3625 .try_claim_for_sequences(
3626 &[&first, &second],
3627 &demand(&[(1, 1), (2, 2)], &[(1, 1), (2, 2)]),
3628 )
3629 .unwrap();
3630 assert!(matches!(deferred, BatchCapacityClaimDecision::Deferred(_)));
3631 let rejected = coordinator
3632 .try_claim_for_sequences(&[&first, &second], &demand(&[(1, 21)], &[(1, 21)]))
3633 .unwrap();
3634 assert!(matches!(
3635 rejected,
3636 BatchCapacityClaimDecision::PermanentRejected(_)
3637 ));
3638 let after = coordinator.snapshot().unwrap();
3639 assert_eq!(before.domains, after.domains);
3640 assert_eq!(after.active_child_claims(), 0);
3641 let state = coordinator.inner.state.lock().unwrap();
3642 assert_eq!(
3643 state.live_sequences[first.sequence().sparse_id() as usize]
3644 .unwrap()
3645 .active_child_claims,
3646 0
3647 );
3648 assert_eq!(
3649 state.live_sequences[second.sequence().sparse_id() as usize]
3650 .unwrap()
3651 .active_child_claims,
3652 0
3653 );
3654 }
3655
3656 #[test]
3657 fn overlapping_batch_children_track_each_parent_without_double_charging() {
3658 let coordinator = coordinator(3);
3659 let request = request(&coordinator);
3660 let first = admit_sequence(&coordinator, &request, &empty_demand());
3661 let second = admit_sequence(&coordinator, &request, &empty_demand());
3662 let third = admit_sequence(&coordinator, &request, &empty_demand());
3663 let first_batch = claimed_batch(
3664 coordinator
3665 .try_claim_for_sequences(&[&first, &second], &demand(&[(1, 1)], &[(1, 1)]))
3666 .unwrap(),
3667 );
3668 let second_batch = claimed_batch(
3669 coordinator
3670 .try_claim_for_sequences(&[&second, &third], &demand(&[(1, 1)], &[(1, 1)]))
3671 .unwrap(),
3672 );
3673 let snapshot = coordinator.snapshot().unwrap();
3674 assert_eq!(snapshot.active_child_claims(), 2);
3675 assert_eq!(snapshot.domains()[0].used().get(), 2);
3676 let state = coordinator.inner.state.lock().unwrap();
3677 let child_counts = [&first, &second, &third].map(|parent| {
3678 state.live_sequences[parent.sequence().sparse_id() as usize]
3679 .unwrap()
3680 .active_child_claims
3681 });
3682 assert_eq!(child_counts, [1, 2, 1]);
3683 drop(state);
3684 drop(first_batch);
3685 assert_eq!(coordinator.snapshot().unwrap().active_child_claims(), 1);
3686 drop(second_batch);
3687 let released = coordinator.snapshot().unwrap();
3688 assert_eq!(released.active_child_claims(), 0);
3689 assert_eq!(released.domains()[0].used().get(), 0);
3690 }
3691
3692 #[test]
3693 fn batch_child_unwind_releases_all_parent_edges() {
3694 let coordinator = coordinator(2);
3695 let request = request(&coordinator);
3696 let first = admit_sequence(&coordinator, &request, &empty_demand());
3697 let second = admit_sequence(&coordinator, &request, &empty_demand());
3698 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
3699 let _batch = claimed_batch(
3700 coordinator
3701 .try_claim_for_sequences(&[&first, &second], &demand(&[(1, 2)], &[(1, 2)]))
3702 .unwrap(),
3703 );
3704 panic!("inject batch invocation cancellation unwind");
3705 }));
3706 assert!(result.is_err());
3707 let snapshot = coordinator.snapshot().unwrap();
3708 assert!(!snapshot.poisoned());
3709 assert_eq!(snapshot.active_child_claims(), 0);
3710 assert_eq!(snapshot.domains()[0].used().get(), 0);
3711 let state = coordinator.inner.state.lock().unwrap();
3712 assert!([&first, &second].iter().all(|parent| state.live_sequences
3713 [parent.sequence().sparse_id() as usize]
3714 .unwrap()
3715 .active_child_claims
3716 == 0));
3717 }
3718
3719 #[test]
3720 fn batch_child_rejects_stale_later_parent_without_partial_effect() {
3721 let coordinator = coordinator(2);
3722 let request = request(&coordinator);
3723 let first = admit_sequence(&coordinator, &request, &empty_demand());
3724 let second = admit_sequence(&coordinator, &request, &empty_demand());
3725 {
3726 let mut state = coordinator.inner.state.lock().unwrap();
3727 state.live_sequences[second.sequence().sparse_id() as usize]
3728 .as_mut()
3729 .unwrap()
3730 .generation += 1;
3731 }
3732 let before = coordinator.snapshot().unwrap();
3733 assert!(coordinator
3734 .try_claim_for_sequences(&[&first, &second], &demand(&[(1, 2)], &[(1, 2)]))
3735 .is_err());
3736 let after = coordinator.snapshot().unwrap();
3737 assert_eq!(before.domains, after.domains);
3738 assert_eq!(after.active_child_claims(), 0);
3739 {
3740 let mut state = coordinator.inner.state.lock().unwrap();
3741 state.live_sequences[second.sequence().sparse_id() as usize]
3742 .as_mut()
3743 .unwrap()
3744 .generation = second.sequence().generation();
3745 }
3746 }
3747
3748 #[test]
3749 fn early_release_of_any_batch_parent_fails_closed_and_retains_shared_claim() {
3750 let coordinator = coordinator(2);
3751 let request = request(&coordinator);
3752 let first = admit_sequence(&coordinator, &request, &empty_demand());
3753 let mut second = admit_sequence(&coordinator, &request, &empty_demand());
3754 let batch = claimed_batch(
3755 coordinator
3756 .try_claim_for_sequences(&[&first, &second], &demand(&[(1, 2)], &[(1, 2)]))
3757 .unwrap(),
3758 );
3759 assert!(!second.release_inner());
3760 let snapshot = coordinator.snapshot().unwrap();
3761 assert!(snapshot.poisoned());
3762 assert_eq!(snapshot.active_sequences(), 2);
3763 assert_eq!(snapshot.active_child_claims(), 1);
3764 assert_eq!(snapshot.domains()[0].used().get(), 2);
3765 drop(batch);
3766 drop(first);
3767 drop(second);
3768 drop(request);
3769 }
3770
3771 #[test]
3772 fn child_multi_domain_defer_and_reject_have_zero_partial_effect() {
3773 let coordinator = coordinator(2);
3774 let request = request(&coordinator);
3775 let parent = admit_sequence(
3776 &coordinator,
3777 &request,
3778 &demand(&[(1, 2), (2, 3)], &[(1, 2), (2, 3)]),
3779 );
3780 let before = coordinator.snapshot().unwrap();
3781 let deferred = coordinator
3782 .try_claim_for_sequence(&parent, &demand(&[(1, 2), (2, 2)], &[(1, 2), (2, 2)]))
3783 .unwrap();
3784 assert!(matches!(deferred, CapacityClaimDecision::Deferred(_)));
3785 let after_defer = coordinator.snapshot().unwrap();
3786 assert_eq!(before.domains, after_defer.domains);
3787 assert_eq!(after_defer.active_child_claims(), 0);
3788
3789 let rejected = coordinator
3790 .try_claim_for_sequence(&parent, &demand(&[(1, 21), (2, 1)], &[(1, 21), (2, 1)]))
3791 .unwrap();
3792 assert!(matches!(
3793 rejected,
3794 CapacityClaimDecision::PermanentRejected(_)
3795 ));
3796 let after_reject = coordinator.snapshot().unwrap();
3797 assert_eq!(before.domains, after_reject.domains);
3798 assert_eq!(after_reject.active_child_claims(), 0);
3799 }
3800
3801 #[test]
3802 fn concurrent_children_preserve_global_and_per_parent_counts() {
3803 const WORKER_COUNT: usize = 3;
3804 const _: () = assert!(WORKER_COUNT <= TEST_NATIVE_WORKER_LIMIT);
3805
3806 let coordinator = coordinator(2);
3807 let request = Arc::new(request(&coordinator));
3808 let first = Arc::new(admit_sequence(
3809 &coordinator,
3810 &request,
3811 &demand(&[(1, 1)], &[(1, 1)]),
3812 ));
3813 let second = Arc::new(admit_sequence(
3814 &coordinator,
3815 &request,
3816 &demand(&[(1, 1)], &[(1, 1)]),
3817 ));
3818 let gate = CancellableTestGate::new(WORKER_COUNT);
3819 thread::scope(|scope| {
3820 let mut cancel_on_unwind = CancelTestGateOnDrop::new(&gate);
3821 let parents = [Arc::clone(&first), Arc::clone(&first), Arc::clone(&second)];
3822 let mut handles: Vec<thread::ScopedJoinHandle<'_, ()>> =
3823 Vec::with_capacity(WORKER_COUNT);
3824 for (worker_index, parent) in parents.into_iter().enumerate() {
3825 let worker_gate = Arc::clone(&gate);
3826 let coordinator = &coordinator;
3827 let handle = match thread::Builder::new()
3828 .name(format!("admission-child-{worker_index}"))
3829 .spawn_scoped(scope, move || {
3830 cancel_gate_on_unwind(&worker_gate, || {
3831 let child = claimed_child(
3832 coordinator
3833 .try_claim_for_sequence(&parent, &demand(&[(1, 1)], &[(1, 1)]))
3834 .unwrap(),
3835 );
3836 let _ = worker_gate.arrive_and_wait();
3837 drop(child);
3838 });
3839 }) {
3840 Ok(handle) => handle,
3841 Err(error) => {
3842 gate.cancel();
3843 for handle in handles.drain(..) {
3844 let _ = handle.join();
3845 }
3846 panic!("failed to spawn bounded child worker: {error}");
3847 }
3848 };
3849 handles.push(handle);
3850 }
3851 if !gate.wait_until_ready(WORKER_COUNT) {
3852 gate.cancel();
3853 for handle in handles.drain(..) {
3854 let _ = handle.join();
3855 }
3856 panic!("bounded child worker cancelled before every worker became ready");
3857 }
3858 let snapshot = coordinator.snapshot().unwrap();
3859 assert_eq!(snapshot.active_requests(), 1);
3860 assert_eq!(snapshot.active_sequences(), 2);
3861 assert_eq!(snapshot.active_child_claims(), 3);
3862 let state = coordinator.inner.state.lock().unwrap();
3863 assert_eq!(
3864 state.live_sequences[first.sequence().sparse_id() as usize]
3865 .unwrap()
3866 .active_child_claims,
3867 2
3868 );
3869 assert_eq!(
3870 state.live_sequences[second.sequence().sparse_id() as usize]
3871 .unwrap()
3872 .active_child_claims,
3873 1
3874 );
3875 drop(state);
3876 gate.release();
3877 cancel_on_unwind.disarm();
3878 for handle in handles {
3879 handle.join().unwrap();
3880 }
3881 });
3882 assert_eq!(coordinator.snapshot().unwrap().active_child_claims(), 0);
3883 }
3884
3885 #[test]
3886 fn child_unwind_releases_without_poisoning_parent() {
3887 let coordinator = coordinator(1);
3888 let request = request(&coordinator);
3889 let parent = admit_sequence(&coordinator, &request, &demand(&[(1, 1)], &[(1, 1)]));
3890 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
3891 let _child = claimed_child(
3892 coordinator
3893 .try_claim_for_sequence(&parent, &demand(&[(1, 2)], &[(1, 2)]))
3894 .unwrap(),
3895 );
3896 panic!("inject invocation cancellation unwind");
3897 }));
3898 assert!(result.is_err());
3899 let snapshot = coordinator.snapshot().unwrap();
3900 assert!(!snapshot.poisoned());
3901 assert_eq!(snapshot.active_sequences(), 1);
3902 assert_eq!(snapshot.active_child_claims(), 0);
3903 assert_eq!(snapshot.domains()[0].used().get(), 1);
3904 }
3905
3906 #[test]
3907 fn child_claim_rejects_stale_sequence_generation_without_side_effect() {
3908 let coordinator = coordinator(1);
3909 let request = request(&coordinator);
3910 let parent = admit_sequence(&coordinator, &request, &demand(&[(1, 1)], &[(1, 1)]));
3911 {
3912 let mut state = coordinator.inner.state.lock().unwrap();
3913 state.live_sequences[parent.sequence().sparse_id() as usize]
3914 .as_mut()
3915 .unwrap()
3916 .generation += 1;
3917 }
3918 let before = coordinator.snapshot().unwrap();
3919 assert!(coordinator
3920 .try_claim_for_sequence(&parent, &demand(&[(1, 1)], &[(1, 1)]))
3921 .is_err());
3922 let after = coordinator.snapshot().unwrap();
3923 assert_eq!(before.domains, after.domains);
3924 assert_eq!(after.active_child_claims(), 0);
3925 {
3926 let mut state = coordinator.inner.state.lock().unwrap();
3927 state.live_sequences[parent.sequence().sparse_id() as usize]
3928 .as_mut()
3929 .unwrap()
3930 .generation = parent.sequence().generation();
3931 }
3932 }
3933
3934 #[test]
3935 fn child_claim_is_counted_in_future_request_epoch_headroom() {
3936 let coordinator = coordinator(1);
3937 let request = request(&coordinator);
3938 let parent = admit_sequence(&coordinator, &request, &demand(&[(1, 1)], &[(1, 1)]));
3939 let child = claimed_child(
3940 coordinator
3941 .try_claim_for_sequence(&parent, &demand(&[(1, 1)], &[(1, 1)]))
3942 .unwrap(),
3943 );
3944 {
3945 let mut state = coordinator.inner.state.lock().unwrap();
3946 state.release_epoch = u64::MAX - 3;
3947 }
3948 let before = coordinator.snapshot().unwrap();
3949 assert!(matches!(
3950 coordinator.try_admit_request(&empty_demand()),
3951 Err(VNextError::DynamicAdmissionContract {
3952 kind: DynamicAdmissionFaultKind::EpochExhausted,
3953 ..
3954 })
3955 ));
3956 let after = coordinator.snapshot().unwrap();
3957 assert_eq!(before.active_requests(), after.active_requests());
3958 assert_eq!(before.active_sequences(), after.active_sequences());
3959 assert_eq!(before.active_child_claims(), after.active_child_claims());
3960 assert_eq!(before.domains, after.domains);
3961 drop(child);
3962 drop(parent);
3963 drop(request);
3964 assert_eq!(coordinator.epochs().unwrap().release_epoch(), u64::MAX);
3965 }
3966
3967 #[test]
3968 fn early_sequence_release_with_active_child_fails_closed() {
3969 let coordinator = coordinator(1);
3970 let request = request(&coordinator);
3971 let mut parent = admit_sequence(&coordinator, &request, &demand(&[(1, 1)], &[(1, 1)]));
3972 let child = claimed_child(
3973 coordinator
3974 .try_claim_for_sequence(&parent, &demand(&[(1, 1)], &[(1, 1)]))
3975 .unwrap(),
3976 );
3977 assert!(!parent.release_inner());
3978 let snapshot = coordinator.snapshot().unwrap();
3979 assert!(snapshot.poisoned());
3980 assert_eq!(snapshot.active_sequences(), 1);
3981 assert_eq!(snapshot.active_child_claims(), 1);
3982 drop(child);
3983 drop(parent);
3984 drop(request);
3985 }
3986
3987 #[test]
3988 fn poisoned_child_drop_retains_capacity_and_child_counts() {
3989 let coordinator = coordinator(1);
3990 let request = request(&coordinator);
3991 let parent = admit_sequence(&coordinator, &request, &demand(&[(1, 1)], &[(1, 1)]));
3992 let child = claimed_child(
3993 coordinator
3994 .try_claim_for_sequence(&parent, &demand(&[(1, 2)], &[(1, 2)]))
3995 .unwrap(),
3996 );
3997 let inner = Arc::clone(&coordinator.inner);
3998 let _ = thread::spawn(move || {
3999 let _guard = inner.state.lock().unwrap();
4000 panic!("poison coordinator before child release");
4001 })
4002 .join();
4003 drop(child);
4004 let snapshot = coordinator.snapshot().unwrap();
4005 assert!(snapshot.poisoned());
4006 assert_eq!(snapshot.active_requests(), 1);
4007 assert_eq!(snapshot.active_sequences(), 1);
4008 assert_eq!(snapshot.active_child_claims(), 1);
4009 assert_eq!(snapshot.domains()[0].used().get(), 3);
4010 drop(parent);
4011 drop(request);
4012 }
4013
4014 #[test]
4015 fn child_claim_rejects_foreign_parent_and_epoch_exhaustion_atomically() {
4016 let local = coordinator(2);
4017 let foreign = coordinator(2);
4018 let foreign_request = request(&foreign);
4019 let foreign_parent = admit_sequence(
4020 &foreign,
4021 &foreign_request,
4022 &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)]),
4023 );
4024 assert!(matches!(
4025 local.try_claim_for_sequence(
4026 &foreign_parent,
4027 &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)])
4028 ),
4029 Err(VNextError::DynamicAdmissionContract {
4030 kind: DynamicAdmissionFaultKind::ForeignCoordinator,
4031 ..
4032 })
4033 ));
4034
4035 let local_request = request(&local);
4036 let parent = admit_sequence(
4037 &local,
4038 &local_request,
4039 &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)]),
4040 );
4041 {
4042 let mut state = local.inner.state.lock().unwrap();
4043 state.release_epoch = u64::MAX - 1;
4044 }
4045 let before = local.snapshot().unwrap();
4046 assert!(matches!(
4047 local.try_claim_for_sequence(&parent, &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)])),
4048 Err(VNextError::DynamicAdmissionContract {
4049 kind: DynamicAdmissionFaultKind::EpochExhausted,
4050 ..
4051 })
4052 ));
4053 let after = local.snapshot().unwrap();
4054 assert_eq!(before.domains, after.domains);
4055 assert_eq!(after.active_child_claims(), 0);
4056 }
4057
4058 #[test]
4059 fn multi_domain_failure_has_zero_partial_effect() {
4060 let coordinator = coordinator(8);
4061 let request = request(&coordinator);
4062 let before = coordinator.snapshot().unwrap();
4063 let decision = coordinator
4064 .try_admit_sequence_for_request(&request, &demand(&[(1, 3), (2, 4)], &[(1, 3), (2, 5)]))
4065 .unwrap();
4066 assert!(matches!(decision, AdmissionDecision::PermanentRejected(_)));
4067 let after = coordinator.snapshot().unwrap();
4068 assert_eq!(before.domains, after.domains);
4069 assert_eq!(after.active_sequences(), 0);
4070 assert_eq!(after.live_sequence_records(), 0);
4071 }
4072
4073 #[test]
4074 fn temporary_multi_domain_shortfall_has_zero_partial_effect() {
4075 let coordinator = coordinator(8);
4076 let request = request(&coordinator);
4077 let held = admit_sequence(
4078 &coordinator,
4079 &request,
4080 &demand(&[(1, 1), (2, 3)], &[(1, 1), (2, 3)]),
4081 );
4082 let before = coordinator.snapshot().unwrap();
4083 let decision = coordinator
4084 .try_admit_sequence_for_request(&request, &demand(&[(1, 2), (2, 2)], &[(1, 2), (2, 2)]))
4085 .unwrap();
4086 assert!(matches!(decision, AdmissionDecision::Deferred(_)));
4087 let after = coordinator.snapshot().unwrap();
4088 assert_eq!(before.domains, after.domains);
4089 assert_eq!(before.active_sequences(), after.active_sequences());
4090 drop(held);
4091 }
4092
4093 #[test]
4094 fn growth_defer_and_permanent_reject_are_distinct() {
4095 let coordinator = coordinator(8);
4096 let request = request(&coordinator);
4097 let growth = coordinator
4098 .try_admit_sequence_for_request(
4099 &request,
4100 &demand(&[(1, 11), (2, 1)], &[(1, 11), (2, 1)]),
4101 )
4102 .unwrap();
4103 assert!(matches!(
4104 growth,
4105 AdmissionDecision::Deferred(AdmissionDeferred {
4106 action: DeferredAction::AwaitBackingGrowth,
4107 ..
4108 })
4109 ));
4110 let impossible = coordinator
4111 .try_admit_sequence_for_request(
4112 &request,
4113 &demand(&[(1, 21), (2, 1)], &[(1, 21), (2, 1)]),
4114 )
4115 .unwrap();
4116 assert!(matches!(
4117 impossible,
4118 AdmissionDecision::PermanentRejected(_)
4119 ));
4120 assert!(matches!(
4121 coordinator
4122 .try_admit_sequence_for_request(
4123 &request,
4124 &demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]),
4125 )
4126 .unwrap(),
4127 AdmissionDecision::Admitted(_)
4128 ));
4129 }
4130
4131 #[test]
4132 fn lease_authority_is_bound_to_the_exact_coordinator() {
4133 let coordinator_a = coordinator(8);
4134 let coordinator_b = coordinator(8);
4135 assert_ne!(coordinator_a.id(), coordinator_b.id());
4136 let request = request(&coordinator_a);
4137 let lease = admit_sequence(
4138 &coordinator_a,
4139 &request,
4140 &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)]),
4141 );
4142 assert!(coordinator_a.owns(&lease));
4143 assert!(!coordinator_b.owns(&lease));
4144 }
4145
4146 #[test]
4147 fn sparse_sequence_ids_reuse_storage_and_change_generation() {
4148 let coordinator = coordinator(u32::MAX);
4149 let parent_request = request(&coordinator);
4150 let sequence_demand = demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)]);
4151 let mut previous_generation = 0;
4152 for _ in 0..1_000_000 {
4153 let lease = admit_sequence(&coordinator, &parent_request, &sequence_demand);
4154 assert_eq!(lease.sequence().sparse_id(), 0);
4155 assert!(lease.sequence().generation() > previous_generation);
4156 previous_generation = lease.sequence().generation();
4157 drop(lease);
4158 }
4159 let snapshot = coordinator.snapshot().unwrap();
4160 assert_eq!(snapshot.active_sequences(), 0);
4161 assert_eq!(snapshot.live_sequence_records(), 0);
4162 assert_eq!(snapshot.reusable_sequence_ids(), 1);
4163 assert_eq!(snapshot.release_epoch(), 1_000_001);
4164 assert_eq!(snapshot.capacity_epoch(), 1);
4165 }
4166
4167 #[test]
4168 fn lease_release_uses_preallocated_reuse_storage() {
4169 let coordinator = coordinator(1);
4170 let request = request(&coordinator);
4171 let lease = admit_sequence(
4172 &coordinator,
4173 &request,
4174 &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)]),
4175 );
4176 let before = {
4177 let state = coordinator.inner.state.lock().unwrap();
4178 assert!(state.reusable_sequence_ids.capacity() >= state.live_sequences.len());
4179 (
4180 state.live_sequences.capacity(),
4181 state.reusable_sequence_ids.capacity(),
4182 )
4183 };
4184 drop(lease);
4185 let state = coordinator.inner.state.lock().unwrap();
4186 assert_eq!(state.live_sequences.capacity(), before.0);
4187 assert_eq!(state.reusable_sequence_ids.capacity(), before.1);
4188 assert_eq!(state.reusable_sequence_ids.as_slice(), &[0]);
4189 }
4190
4191 #[test]
4192 fn snapshot_counts_live_authorities_independently_from_active_counter() {
4193 let coordinator = coordinator(1);
4194 let mut state = coordinator.inner.state.lock().unwrap();
4195 state.active_sequences = 1;
4196 state.poisoned = true;
4197 let snapshot = state.snapshot(coordinator.id());
4198 assert_eq!(snapshot.active_sequences(), 1);
4199 assert_eq!(snapshot.live_sequence_records(), 0);
4200 }
4201
4202 #[test]
4203 fn concurrent_admission_never_exceeds_ceiling() {
4204 const SEQUENCE_CEILING: u32 = 4;
4205 const WORKER_COUNT: usize = 8;
4206 const _: () = assert!(WORKER_COUNT <= TEST_NATIVE_WORKER_LIMIT);
4207
4208 let coordinator = Arc::new(coordinator(SEQUENCE_CEILING));
4209 let parent_request = Arc::new(request(&coordinator));
4210 let gate = CancellableTestGate::new(WORKER_COUNT);
4211 let mut handles: Vec<thread::JoinHandle<Option<AdmissionDecision>>> =
4212 Vec::with_capacity(WORKER_COUNT);
4213 for worker_index in 0..WORKER_COUNT {
4214 let coordinator = Arc::clone(&coordinator);
4215 let parent_request = Arc::clone(&parent_request);
4216 let worker_gate = Arc::clone(&gate);
4217 let handle = match thread::Builder::new()
4218 .name(format!("admission-sequence-{worker_index}"))
4219 .spawn(move || {
4220 cancel_gate_on_unwind(&worker_gate, || {
4221 if !worker_gate.arrive_and_wait() {
4222 return None;
4223 }
4224 Some(
4225 coordinator
4226 .try_admit_sequence_for_request(&parent_request, &empty_demand())
4227 .unwrap(),
4228 )
4229 })
4230 }) {
4231 Ok(handle) => handle,
4232 Err(error) => {
4233 gate.cancel();
4234 for handle in handles.drain(..) {
4235 let _ = handle.join();
4236 }
4237 panic!("failed to spawn bounded admission worker: {error}");
4238 }
4239 };
4240 handles.push(handle);
4241 }
4242 if !gate.wait_until_ready(WORKER_COUNT) {
4243 gate.cancel();
4244 for handle in handles.drain(..) {
4245 let _ = handle.join();
4246 }
4247 panic!("bounded admission worker cancelled before every worker became ready");
4248 }
4249 gate.release();
4250 let mut leases = Vec::new();
4251 for handle in handles {
4252 if let Some(AdmissionDecision::Admitted(lease)) = handle.join().unwrap() {
4253 leases.push(lease);
4254 }
4255 }
4256 assert_eq!(leases.len(), SEQUENCE_CEILING as usize);
4257 assert_eq!(
4258 coordinator.snapshot().unwrap().active_sequences(),
4259 SEQUENCE_CEILING
4260 );
4261
4262 let preempt_demand = AdmissionDemand::from_plan(
4263 CapacityVector::empty(),
4264 CapacityVector::empty(),
4265 AdmissionFitPolicy::ImmediateOnly,
4266 AdmissionPressureAction::PreemptAndRecompute,
4267 )
4268 .unwrap();
4269 let preflight = coordinator
4270 .preflight_sequence_ceiling_for_request(&parent_request, &preempt_demand)
4271 .unwrap();
4272 match preflight {
4273 AdmissionPreflightDecision::Deferred(deferred) => {
4274 assert_eq!(deferred.action(), DeferredAction::PreemptAndRecompute);
4275 assert!(deferred
4276 .blockers()
4277 .iter()
4278 .any(|blocker| blocker.kind() == CapacityShortfallKind::ActiveSequenceCeiling));
4279 }
4280 _ => panic!("full sequence ceiling must defer preflight"),
4281 }
4282
4283 for (pressure_action, expected) in [
4284 (
4285 AdmissionPressureAction::WaitForRelease,
4286 DeferredAction::WaitForRelease,
4287 ),
4288 (
4289 AdmissionPressureAction::PreemptAndRecompute,
4290 DeferredAction::PreemptAndRecompute,
4291 ),
4292 ] {
4293 let growth_demand = AdmissionDemand::from_plan(
4294 vector(&[(1, 11)]),
4295 vector(&[(1, 11)]),
4296 AdmissionFitPolicy::ImmediateOnly,
4297 pressure_action,
4298 )
4299 .unwrap();
4300 let decision = coordinator
4301 .preflight_sequence_ceiling_for_request(&parent_request, &growth_demand)
4302 .unwrap();
4303 match decision {
4304 AdmissionPreflightDecision::Deferred(deferred) => {
4305 assert_eq!(deferred.action(), expected);
4306 assert!(deferred.blockers().iter().any(|blocker| {
4307 blocker.kind() == CapacityShortfallKind::ActiveSequenceCeiling
4308 }));
4309 assert!(deferred.blockers().iter().any(|blocker| {
4310 blocker.kind() == CapacityShortfallKind::BackingGrowthRequired
4311 }));
4312 }
4313 _ => panic!("mixed ceiling and growth pressure must remain a logical defer"),
4314 }
4315 }
4316 drop(leases);
4317 assert_eq!(coordinator.snapshot().unwrap().active_sequences(), 0);
4318 }
4319
4320 #[test]
4321 fn zero_domain_plan_still_uses_dynamic_sequence_authority() {
4322 let coordinator = LogicalAdmissionCoordinator::new(Vec::new(), u32::MAX).unwrap();
4323 let parent_request = request(&coordinator);
4324 let sequence_demand = empty_demand();
4325 let lease = admit_sequence(&coordinator, &parent_request, &sequence_demand);
4326 assert!(lease.claims().is_empty());
4327 assert_eq!(coordinator.snapshot().unwrap().active_sequences(), 1);
4328 drop(lease);
4329 assert_eq!(coordinator.snapshot().unwrap().active_sequences(), 0);
4330 }
4331
4332 #[test]
4333 fn sequence_issue_failure_has_zero_side_effect() {
4334 let coordinator = coordinator(8);
4335 let request = request(&coordinator);
4336 {
4337 let mut state = coordinator.inner.state.lock().unwrap();
4338 state.next_sequence_generation = u64::MAX;
4339 }
4340 let before = coordinator.snapshot().unwrap();
4341 assert!(
4342 coordinator
4343 .try_admit_sequence_for_request(
4344 &request,
4345 &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)]),
4346 )
4347 .is_err()
4348 );
4349 let after = coordinator.snapshot().unwrap();
4350 assert_eq!(before.domains, after.domains);
4351 assert_eq!(after.active_sequences(), 0);
4352 assert_eq!(after.live_sequence_records(), 0);
4353 }
4354
4355 #[test]
4356 fn epoch_exhaustion_rejects_admission_before_claim() {
4357 let coordinator = coordinator(8);
4358 let request = request(&coordinator);
4359 {
4360 let mut state = coordinator.inner.state.lock().unwrap();
4361 state.release_epoch = u64::MAX;
4362 }
4363 let before = coordinator.snapshot().unwrap();
4364 assert!(
4365 coordinator
4366 .try_admit_sequence_for_request(
4367 &request,
4368 &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)]),
4369 )
4370 .is_err()
4371 );
4372 let after = coordinator.snapshot().unwrap();
4373 assert_eq!(before.domains, after.domains);
4374 assert_eq!(after.active_sequences(), 0);
4375 }
4376
4377 #[test]
4378 fn waiter_recheck_closes_release_and_growth_races() {
4379 let coordinator = coordinator(1);
4380 let request = request(&coordinator);
4381 let observed = wait_for_domains(&coordinator, &[1]);
4382 let registration = coordinator.register_waiter(observed).unwrap();
4383 assert!(!registration.recheck().unwrap().should_retry());
4384
4385 coordinator
4386 .set_domain_total(domain(1), CapacityUnits::new(11))
4387 .unwrap();
4388 let recheck = registration.recheck().unwrap();
4389 assert!(recheck.changed_since_registration());
4390 assert_eq!(recheck.current().capacity_epoch(), 2);
4391
4392 let lease = admit_sequence(
4393 &coordinator,
4394 &request,
4395 &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)]),
4396 );
4397 let before_release = wait_for_domains(&coordinator, &[1]);
4398 drop(lease);
4399 let late_registration = coordinator.register_waiter(before_release).unwrap();
4400 assert!(late_registration
4401 .recheck()
4402 .unwrap()
4403 .changed_since_observation());
4404 assert_eq!(coordinator.epochs().unwrap().capacity_epoch(), 2);
4405 }
4406
4407 #[test]
4408 fn waiter_retries_only_when_its_exact_domain_changes() {
4409 let coordinator = coordinator(8);
4410 let observed = wait_for_domains(&coordinator, &[1]);
4411 let registration = coordinator.register_waiter(observed).unwrap();
4412
4413 let before = coordinator.epochs().unwrap();
4414 coordinator
4415 .notify_domain_availability_changed(domain(2))
4416 .unwrap();
4417 let unrelated = registration.recheck().unwrap();
4418 assert_eq!(
4419 unrelated.current().capacity_epoch(),
4420 before.capacity_epoch() + 1
4421 );
4422 assert!(!unrelated.changed_since_observation());
4423 assert!(!unrelated.changed_since_registration());
4424 assert!(!unrelated.should_retry());
4425
4426 coordinator
4427 .notify_domain_availability_changed(domain(1))
4428 .unwrap();
4429 let relevant = registration.recheck().unwrap();
4430 assert!(relevant.changed_since_observation());
4431 assert!(relevant.changed_since_registration());
4432 assert!(relevant.should_retry());
4433 }
4434
4435 #[test]
4436 fn wait_snapshot_keeps_pre_mutation_audit_and_exact_source_together() {
4437 let coordinator = coordinator(8);
4438 let snapshot = coordinator
4439 .wait_snapshot_for_domains([domain(1), domain(2)])
4440 .unwrap()
4441 .narrow_to_domains([domain(1)])
4442 .unwrap();
4443 let observed_epochs = snapshot.epochs();
4444
4445 coordinator
4446 .notify_domain_availability_changed(domain(1))
4447 .unwrap();
4448 let mut current = Vec::new();
4449 let current_epochs = coordinator.write_availability_epochs(&mut current).unwrap();
4450
4451 assert_eq!(
4452 snapshot.wait_condition().observed(),
4453 &[
4454 CapacityAvailabilityEpoch::new(CapacityAvailabilitySource::Domain(domain(1)), 1,)
4455 .unwrap()
4456 ]
4457 );
4458 assert_eq!(observed_epochs.capacity_epoch(), 1);
4459 assert_eq!(current_epochs.capacity_epoch(), 2);
4460 assert!(snapshot.wait_condition().changed_since(¤t).unwrap());
4461 }
4462
4463 #[test]
4464 fn active_sequence_waiter_retries_when_a_slot_is_released() {
4465 let coordinator = coordinator(1);
4466 let first_request = request(&coordinator);
4467 let first = admit_sequence(&coordinator, &first_request, &empty_demand());
4468 let second_request = request(&coordinator);
4469 let deferred = match coordinator
4470 .try_admit_sequence_for_request(&second_request, &empty_demand())
4471 .unwrap()
4472 {
4473 AdmissionDecision::Deferred(deferred) => deferred,
4474 _ => panic!("active-sequence ceiling must defer the second sequence"),
4475 };
4476 assert_eq!(
4477 deferred.wait_condition().observed()[0].source(),
4478 CapacityAvailabilitySource::ActiveSequenceSlots
4479 );
4480 let registration = coordinator
4481 .register_waiter(deferred.wait_condition().clone())
4482 .unwrap();
4483 assert!(!registration.recheck().unwrap().should_retry());
4484
4485 drop(first);
4486 assert!(registration.recheck().unwrap().should_retry());
4487 }
4488
4489 #[test]
4490 fn changed_source_cannot_hide_another_source_regression() {
4491 let first = CapacityAvailabilitySource::Domain(domain(1));
4492 let second = CapacityAvailabilitySource::Domain(domain(2));
4493 let observed = CapacityWaitCondition::from_observation(
4494 41,
4495 vec![
4496 CapacityAvailabilityEpoch::new(first, 5).unwrap(),
4497 CapacityAvailabilityEpoch::new(second, 5).unwrap(),
4498 ],
4499 )
4500 .unwrap();
4501 let current = vec![
4502 CapacityAvailabilityEpoch::new(first, 6).unwrap(),
4503 CapacityAvailabilityEpoch::new(second, 4).unwrap(),
4504 ];
4505
4506 assert!(matches!(
4507 observed.changed_since(¤t),
4508 Err(VNextError::DynamicAdmissionContract {
4509 kind: DynamicAdmissionFaultKind::EpochRegression,
4510 ..
4511 })
4512 ));
4513 }
4514
4515 #[test]
4516 fn allocator_availability_change_advances_capacity_epoch_without_recounting_units() {
4517 let coordinator = coordinator(8);
4518 let before = coordinator.snapshot().unwrap();
4519 let observed = wait_for_domains(&coordinator, &[1]);
4520 let registration = coordinator.register_waiter(observed).unwrap();
4521
4522 let changed = coordinator
4523 .notify_domain_availability_changed(domain(1))
4524 .unwrap();
4525 let after = coordinator.snapshot().unwrap();
4526 assert_eq!(changed.capacity_epoch(), before.capacity_epoch() + 1);
4527 assert_eq!(changed.release_epoch(), before.release_epoch());
4528 assert_eq!(after.domains(), before.domains());
4529 assert!(registration.recheck().unwrap().should_retry());
4530
4531 let before_reject = coordinator.epochs().unwrap();
4532 assert!(coordinator
4533 .notify_domain_availability_changed(domain(99))
4534 .is_err());
4535 assert_eq!(coordinator.epochs().unwrap(), before_reject);
4536 }
4537
4538 #[test]
4539 fn multi_domain_growth_validates_all_before_one_epoch_commit() {
4540 let coordinator = coordinator(8);
4541 let before = coordinator.snapshot().unwrap();
4542 assert!(coordinator
4543 .set_domain_totals(&[
4544 (domain(1), CapacityUnits::new(11)),
4545 (domain(2), CapacityUnits::new(5)),
4546 ])
4547 .is_err());
4548 let rejected = coordinator.snapshot().unwrap();
4549 assert_eq!(before.domains, rejected.domains);
4550 assert_eq!(before.capacity_epoch(), rejected.capacity_epoch());
4551
4552 let committed = coordinator
4553 .set_domain_totals(&[
4554 (domain(1), CapacityUnits::new(11)),
4555 (domain(2), CapacityUnits::new(4)),
4556 ])
4557 .unwrap();
4558 assert_eq!(committed.capacity_epoch(), before.capacity_epoch() + 1);
4559 assert_eq!(
4560 coordinator.snapshot().unwrap().domains()[0].total().get(),
4561 11
4562 );
4563 }
4564
4565 #[tokio::test]
4566 async fn waiter_listener_closes_recheck_to_park_race() {
4567 let coordinator = coordinator(1);
4568 let request = request(&coordinator);
4569 let lease = admit_sequence(
4570 &coordinator,
4571 &request,
4572 &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)]),
4573 );
4574 let observed_epochs = coordinator.epochs().unwrap();
4575 let observed = wait_for_domains(&coordinator, &[1]);
4576 let registration = coordinator.register_waiter(observed).unwrap();
4577 assert!(!registration.recheck().unwrap().should_retry());
4578 drop(lease);
4579 let changed = tokio::time::timeout(
4580 std::time::Duration::from_secs(1),
4581 registration.wait_for_change(),
4582 )
4583 .await
4584 .expect("listener must not miss a release between recheck and park")
4585 .unwrap();
4586 assert!(changed.release_epoch() > observed_epochs.release_epoch());
4587 assert_eq!(changed.coordinator_id(), coordinator.id());
4588 }
4589
4590 #[tokio::test]
4591 async fn mutation_unwind_wakes_parked_waiter_with_terminal_error() {
4592 let coordinator = coordinator(1);
4593 let observed = wait_for_domains(&coordinator, &[1]);
4594 let registration = coordinator.register_waiter(observed).unwrap();
4595 let waiter = tokio::spawn(async move { registration.wait_for_change().await });
4596 tokio::task::yield_now().await;
4597
4598 let inner = Arc::clone(&coordinator.inner);
4599 let _ = thread::spawn(move || {
4600 let _mutation = inner.lock_mutation().unwrap();
4601 panic!("inject mutation unwind");
4602 })
4603 .join();
4604
4605 let result = tokio::time::timeout(std::time::Duration::from_secs(1), waiter)
4606 .await
4607 .expect("mutation unwind must wake an already parked waiter")
4608 .expect("waiter task must not panic");
4609 assert!(matches!(
4610 result,
4611 Err(VNextError::DynamicAdmissionContract {
4612 kind: DynamicAdmissionFaultKind::Poisoned,
4613 ..
4614 })
4615 ));
4616 assert!(coordinator.snapshot().unwrap().poisoned());
4617 }
4618
4619 #[test]
4620 fn sequence_unwind_releases_lease_without_poisoning_coordinator() {
4621 let coordinator = coordinator(1);
4622 let request = request(&coordinator);
4623 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
4624 let _lease = admit_sequence(
4625 &coordinator,
4626 &request,
4627 &demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]),
4628 );
4629 panic!("inject request panic outside coordinator mutation");
4630 }));
4631 assert!(result.is_err());
4632
4633 let snapshot = coordinator.snapshot().unwrap();
4634 assert!(!snapshot.poisoned());
4635 assert_eq!(snapshot.active_sequences(), 0);
4636 assert_eq!(snapshot.live_sequence_records(), 0);
4637 assert!(snapshot
4638 .domains()
4639 .iter()
4640 .all(|domain| domain.used().get() == 0));
4641
4642 let retry = coordinator
4643 .try_admit_sequence_for_request(&request, &demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]))
4644 .unwrap();
4645 assert!(matches!(retry, AdmissionDecision::Admitted(_)));
4646 }
4647
4648 #[test]
4649 fn request_unwind_releases_request_capacity_without_poisoning_coordinator() {
4650 let coordinator = coordinator(1);
4651 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
4652 let _request = admitted_request(
4653 coordinator
4654 .try_admit_request(&demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]))
4655 .unwrap(),
4656 );
4657 panic!("inject request scope unwind");
4658 }));
4659 assert!(result.is_err());
4660 let snapshot = coordinator.snapshot().unwrap();
4661 assert!(!snapshot.poisoned());
4662 assert_eq!(snapshot.active_requests(), 0);
4663 assert_eq!(snapshot.active_sequences(), 0);
4664 assert!(snapshot
4665 .domains()
4666 .iter()
4667 .all(|domain| domain.used().get() == 0));
4668 }
4669
4670 #[test]
4671 fn poisoned_drop_retains_claim_and_is_observable() {
4672 let coordinator = coordinator(8);
4673 let request = request(&coordinator);
4674 let lease = admit_sequence(
4675 &coordinator,
4676 &request,
4677 &demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]),
4678 );
4679 let inner = Arc::clone(&coordinator.inner);
4680 let _ = thread::spawn(move || {
4681 let _guard = inner.state.lock().unwrap();
4682 panic!("poison admission state for conservative-drop test");
4683 })
4684 .join();
4685 drop(lease);
4686 let snapshot = coordinator.snapshot().unwrap();
4687 assert!(snapshot.poisoned());
4688 assert_eq!(snapshot.active_sequences(), 1);
4689 assert_eq!(snapshot.live_sequence_records(), 1);
4690 assert_eq!(snapshot.domains()[0].used().get(), 2);
4691 }
4692}
4693
4694#[cfg(test)]
4695#[path = "admission/model_check.rs"]
4696mod model_check;