1use serde::{Deserialize, Serialize};
2use std::collections::{BTreeMap, BTreeSet};
3use std::ops::{Deref, DerefMut};
4use std::sync::atomic::{AtomicU64, Ordering};
5use std::sync::{Arc, Mutex, MutexGuard};
6use tokio::sync::watch;
7
8use super::{DynamicAdmissionFaultKind, VNextError};
9
10fn invalid_admission(reason: impl Into<String>) -> VNextError {
11 admission_fault(DynamicAdmissionFaultKind::InvalidContract, reason)
12}
13
14fn admission_fault(kind: DynamicAdmissionFaultKind, reason: impl Into<String>) -> VNextError {
15 VNextError::DynamicAdmissionContract {
16 kind,
17 reason: reason.into(),
18 }
19}
20
21static NEXT_COORDINATOR_ID: AtomicU64 = AtomicU64::new(1);
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
24#[serde(transparent)]
25pub struct LogicalAdmissionCoordinatorId(u64);
26
27impl LogicalAdmissionCoordinatorId {
28 fn issue() -> Result<Self, VNextError> {
29 NEXT_COORDINATOR_ID
30 .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
31 current.checked_add(1)
32 })
33 .map(Self)
34 .map_err(|_| {
35 admission_fault(
36 DynamicAdmissionFaultKind::AuthorityExhausted,
37 "logical admission coordinator id is exhausted",
38 )
39 })
40 }
41
42 pub const fn get(self) -> u64 {
43 self.0
44 }
45}
46
47#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
48#[serde(transparent)]
49pub struct CapacityDomainId(u32);
50
51impl CapacityDomainId {
52 pub fn new(value: u32) -> Result<Self, VNextError> {
53 if value == 0 {
54 return Err(invalid_admission("capacity domain id must be non-zero"));
55 }
56 Ok(Self(value))
57 }
58
59 pub const fn get(self) -> u32 {
60 self.0
61 }
62}
63
64#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)]
65#[serde(transparent)]
66pub struct CapacityUnits(u64);
67
68impl CapacityUnits {
69 pub const ZERO: Self = Self(0);
70
71 pub const fn new(value: u64) -> Self {
72 Self(value)
73 }
74
75 pub const fn get(self) -> u64 {
76 self.0
77 }
78}
79
80#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
81pub struct CapacityDomainSpec {
82 total_units: CapacityUnits,
83 maximum_total_units: CapacityUnits,
84}
85
86impl CapacityDomainSpec {
87 pub fn new(
88 total_units: CapacityUnits,
89 maximum_total_units: CapacityUnits,
90 ) -> Result<Self, VNextError> {
91 if total_units.get() > maximum_total_units.get() {
92 return Err(invalid_admission(
93 "capacity domain total exceeds maximum total",
94 ));
95 }
96 Ok(Self {
97 total_units,
98 maximum_total_units,
99 })
100 }
101
102 pub const fn total_units(self) -> CapacityUnits {
103 self.total_units
104 }
105
106 pub const fn maximum_total_units(self) -> CapacityUnits {
107 self.maximum_total_units
108 }
109}
110
111#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
112pub struct CapacityEntry {
113 domain: CapacityDomainId,
114 units: CapacityUnits,
115}
116
117impl CapacityEntry {
118 pub fn new(domain: CapacityDomainId, units: CapacityUnits) -> Result<Self, VNextError> {
119 if units.get() == 0 {
120 return Err(invalid_admission("capacity demand units must be non-zero"));
121 }
122 Ok(Self { domain, units })
123 }
124
125 pub const fn domain(self) -> CapacityDomainId {
126 self.domain
127 }
128
129 pub const fn units(self) -> CapacityUnits {
130 self.units
131 }
132}
133
134#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
135#[serde(transparent)]
136pub struct CapacityVector(Vec<CapacityEntry>);
137
138impl CapacityVector {
139 pub fn new(mut entries: Vec<CapacityEntry>) -> Result<Self, VNextError> {
140 entries.sort_by_key(|entry| entry.domain);
141 if entries.is_empty() {
142 return Err(invalid_admission("capacity vector must be non-empty"));
143 }
144 if entries
145 .windows(2)
146 .any(|pair| pair[0].domain == pair[1].domain)
147 {
148 return Err(invalid_admission(
149 "capacity vector contains a duplicate domain",
150 ));
151 }
152 Ok(Self(entries))
153 }
154
155 pub fn entries(&self) -> &[CapacityEntry] {
156 &self.0
157 }
158
159 pub(crate) const fn empty() -> Self {
160 Self(Vec::new())
161 }
162
163 pub const fn is_empty(&self) -> bool {
164 self.0.is_empty()
165 }
166
167 pub(crate) fn units_for(&self, domain: CapacityDomainId) -> Option<CapacityUnits> {
168 self.0
169 .binary_search_by_key(&domain, |entry| entry.domain)
170 .ok()
171 .map(|index| self.0[index].units)
172 }
173
174 pub(crate) fn checked_sum(left: &Self, right: &Self) -> Result<Self, VNextError> {
175 let mut units_by_domain = BTreeMap::<CapacityDomainId, u64>::new();
176 for entry in left.entries().iter().chain(right.entries()) {
177 let units = units_by_domain.entry(entry.domain()).or_default();
178 *units = units.checked_add(entry.units().get()).ok_or_else(|| {
179 admission_fault(
180 DynamicAdmissionFaultKind::ArithmeticOverflow,
181 "combined admission demand overflows u64",
182 )
183 })?;
184 }
185 if units_by_domain.is_empty() {
186 return Ok(Self::empty());
187 }
188 Self::new(
189 units_by_domain
190 .into_iter()
191 .map(|(domain, units)| CapacityEntry::new(domain, CapacityUnits::new(units)))
192 .collect::<Result<Vec<_>, _>>()?,
193 )
194 }
195}
196
197#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
198#[serde(rename_all = "snake_case")]
199pub enum AdmissionFitPolicy {
200 ImmediateOnly,
201 FullInputMustFit,
204}
205
206#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
207#[serde(rename_all = "snake_case")]
208pub enum AdmissionPressureAction {
209 WaitForRelease,
210 PreemptAndRecompute,
211}
212
213#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
216pub struct AdmissionDemand {
217 immediate_claim: CapacityVector,
218 fit_requirement: CapacityVector,
219 fit_policy: AdmissionFitPolicy,
220 pressure_action: AdmissionPressureAction,
221}
222
223impl AdmissionDemand {
224 pub(crate) fn from_plan(
225 immediate_claim: CapacityVector,
226 fit_requirement: CapacityVector,
227 fit_policy: AdmissionFitPolicy,
228 pressure_action: AdmissionPressureAction,
229 ) -> Result<Self, VNextError> {
230 for immediate in immediate_claim.entries() {
231 let Some(fit_units) = fit_requirement.units_for(immediate.domain) else {
232 return Err(invalid_admission(
233 "fit requirement omits an immediate-claim domain",
234 ));
235 };
236 if fit_units.get() < immediate.units.get() {
237 return Err(invalid_admission(
238 "fit requirement is smaller than the immediate claim",
239 ));
240 }
241 }
242 if fit_policy == AdmissionFitPolicy::ImmediateOnly && fit_requirement != immediate_claim {
243 return Err(invalid_admission(
244 "immediate-only admission requires identical immediate and fit vectors",
245 ));
246 }
247 Ok(Self {
248 immediate_claim,
249 fit_requirement,
250 fit_policy,
251 pressure_action,
252 })
253 }
254
255 pub fn immediate_claim(&self) -> &CapacityVector {
256 &self.immediate_claim
257 }
258
259 pub fn fit_requirement(&self) -> &CapacityVector {
260 &self.fit_requirement
261 }
262
263 pub const fn fit_policy(&self) -> AdmissionFitPolicy {
264 self.fit_policy
265 }
266
267 pub const fn pressure_action(&self) -> AdmissionPressureAction {
268 self.pressure_action
269 }
270
271 fn initial_sequence_bundle(request: &Self, sequence: &Self) -> Result<Self, VNextError> {
272 if request.pressure_action != sequence.pressure_action {
273 return Err(invalid_admission(
274 "initial request and sequence require one pressure action",
275 ));
276 }
277 let fit_policy = if request.fit_policy == AdmissionFitPolicy::FullInputMustFit
278 || sequence.fit_policy == AdmissionFitPolicy::FullInputMustFit
279 {
280 AdmissionFitPolicy::FullInputMustFit
281 } else {
282 AdmissionFitPolicy::ImmediateOnly
283 };
284 Self::from_plan(
285 CapacityVector::checked_sum(&request.immediate_claim, &sequence.immediate_claim)?,
286 CapacityVector::checked_sum(&request.fit_requirement, &sequence.fit_requirement)?,
287 fit_policy,
288 request.pressure_action,
289 )
290 }
291}
292
293#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
294#[serde(rename_all = "snake_case")]
295pub enum CapacityShortfallKind {
296 ImmediateAvailability,
297 FitAvailability,
298 BackingGrowthRequired,
299 ActiveSequenceCeiling,
300 PermanentDomainMaximum,
301}
302
303#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
304pub struct CapacityShortfall {
305 domain: Option<CapacityDomainId>,
306 kind: CapacityShortfallKind,
307 requested: CapacityUnits,
308 available: CapacityUnits,
309 current_total: CapacityUnits,
310 maximum_total: CapacityUnits,
311}
312
313impl CapacityShortfall {
314 pub const fn domain(&self) -> Option<CapacityDomainId> {
315 self.domain
316 }
317
318 pub const fn kind(&self) -> CapacityShortfallKind {
319 self.kind
320 }
321
322 pub const fn requested(&self) -> CapacityUnits {
323 self.requested
324 }
325
326 pub const fn available(&self) -> CapacityUnits {
327 self.available
328 }
329
330 pub const fn current_total(&self) -> CapacityUnits {
331 self.current_total
332 }
333
334 pub const fn maximum_total(&self) -> CapacityUnits {
335 self.maximum_total
336 }
337}
338
339#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
340pub struct DomainCapacitySnapshot {
341 domain: CapacityDomainId,
342 total: CapacityUnits,
343 maximum_total: CapacityUnits,
344 used: CapacityUnits,
345 available: CapacityUnits,
346}
347
348impl DomainCapacitySnapshot {
349 pub const fn domain(&self) -> CapacityDomainId {
350 self.domain
351 }
352
353 pub const fn total(&self) -> CapacityUnits {
354 self.total
355 }
356
357 pub const fn maximum_total(&self) -> CapacityUnits {
358 self.maximum_total
359 }
360
361 pub const fn used(&self) -> CapacityUnits {
362 self.used
363 }
364
365 pub const fn available(&self) -> CapacityUnits {
366 self.available
367 }
368}
369
370#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
371pub struct CapacitySnapshot {
372 coordinator_id: LogicalAdmissionCoordinatorId,
373 domains: Vec<DomainCapacitySnapshot>,
374 active_requests: u32,
375 active_sequences: u32,
376 active_child_claims: u64,
377 maximum_active_sequences: u32,
378 release_epoch: u64,
379 capacity_epoch: u64,
380 live_sequence_records: usize,
381 reusable_sequence_ids: usize,
382 live_request_records: usize,
383 reusable_request_ids: usize,
384 poisoned: bool,
385}
386
387impl CapacitySnapshot {
388 pub const fn coordinator_id(&self) -> LogicalAdmissionCoordinatorId {
389 self.coordinator_id
390 }
391
392 pub fn domains(&self) -> &[DomainCapacitySnapshot] {
393 &self.domains
394 }
395
396 pub const fn active_sequences(&self) -> u32 {
397 self.active_sequences
398 }
399
400 pub const fn active_requests(&self) -> u32 {
401 self.active_requests
402 }
403
404 pub const fn active_child_claims(&self) -> u64 {
405 self.active_child_claims
406 }
407
408 pub const fn maximum_active_sequences(&self) -> u32 {
409 self.maximum_active_sequences
410 }
411
412 pub const fn release_epoch(&self) -> u64 {
413 self.release_epoch
414 }
415
416 pub const fn capacity_epoch(&self) -> u64 {
417 self.capacity_epoch
418 }
419
420 pub const fn live_sequence_records(&self) -> usize {
421 self.live_sequence_records
422 }
423
424 pub const fn reusable_sequence_ids(&self) -> usize {
425 self.reusable_sequence_ids
426 }
427
428 pub const fn live_request_records(&self) -> usize {
429 self.live_request_records
430 }
431
432 pub const fn reusable_request_ids(&self) -> usize {
433 self.reusable_request_ids
434 }
435
436 pub const fn poisoned(&self) -> bool {
437 self.poisoned
438 }
439}
440
441#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
442pub struct CapacityEpochs {
443 coordinator_id: LogicalAdmissionCoordinatorId,
444 release_epoch: u64,
445 capacity_epoch: u64,
446}
447
448impl CapacityEpochs {
449 pub const fn coordinator_id(self) -> LogicalAdmissionCoordinatorId {
450 self.coordinator_id
451 }
452
453 pub const fn release_epoch(self) -> u64 {
454 self.release_epoch
455 }
456
457 pub const fn capacity_epoch(self) -> u64 {
458 self.capacity_epoch
459 }
460}
461
462#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
466#[serde(rename_all = "snake_case")]
467pub enum CapacityAvailabilitySource {
468 Domain(CapacityDomainId),
469 ActiveSequenceSlots,
470 PlanDeviceBudget,
471 ProcessDeviceCapacity,
472}
473
474#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)]
475pub struct CapacityAvailabilityEpoch {
476 source: CapacityAvailabilitySource,
477 epoch: u64,
478}
479
480impl CapacityAvailabilityEpoch {
481 pub fn new(source: CapacityAvailabilitySource, epoch: u64) -> Result<Self, VNextError> {
482 if epoch == 0 {
483 return Err(invalid_admission(
484 "capacity availability epoch must be non-zero",
485 ));
486 }
487 Ok(Self { source, epoch })
488 }
489
490 pub const fn source(self) -> CapacityAvailabilitySource {
491 self.source
492 }
493
494 pub const fn epoch(self) -> u64 {
495 self.epoch
496 }
497}
498
499#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
503pub struct CapacityWaitCondition {
504 coordinator_id: LogicalAdmissionCoordinatorId,
505 observed: Vec<CapacityAvailabilityEpoch>,
506}
507
508impl CapacityWaitCondition {
509 pub fn from_observation(
510 coordinator_id: u64,
511 observed: Vec<CapacityAvailabilityEpoch>,
512 ) -> Result<Self, VNextError> {
513 if coordinator_id == 0 {
514 return Err(invalid_admission(
515 "capacity wait coordinator id must be non-zero",
516 ));
517 }
518 Self::new(LogicalAdmissionCoordinatorId(coordinator_id), observed)
519 }
520
521 pub fn new(
522 coordinator_id: LogicalAdmissionCoordinatorId,
523 mut observed: Vec<CapacityAvailabilityEpoch>,
524 ) -> Result<Self, VNextError> {
525 if observed.is_empty() {
526 return Err(invalid_admission(
527 "capacity wait condition requires at least one availability source",
528 ));
529 }
530 observed.sort_by_key(|entry| entry.source);
531 if observed
532 .windows(2)
533 .any(|pair| pair[0].source == pair[1].source)
534 {
535 return Err(invalid_admission(
536 "capacity wait condition contains a duplicate availability source",
537 ));
538 }
539 Ok(Self {
540 coordinator_id,
541 observed,
542 })
543 }
544
545 pub const fn coordinator_id(&self) -> LogicalAdmissionCoordinatorId {
546 self.coordinator_id
547 }
548
549 pub fn observed(&self) -> &[CapacityAvailabilityEpoch] {
550 &self.observed
551 }
552
553 pub fn validate_sources_present(
554 &self,
555 current: &[CapacityAvailabilityEpoch],
556 ) -> Result<(), VNextError> {
557 if current
558 .windows(2)
559 .any(|pair| pair[0].source >= pair[1].source)
560 {
561 return Err(invalid_admission(
562 "capacity availability snapshot is not canonical",
563 ));
564 }
565 for observed in &self.observed {
566 current
567 .binary_search_by_key(&observed.source, |entry| entry.source)
568 .map_err(|_| {
569 invalid_admission("capacity availability snapshot omitted a waited source")
570 })?;
571 }
572 Ok(())
573 }
574
575 pub fn changed_since(&self, current: &[CapacityAvailabilityEpoch]) -> Result<bool, VNextError> {
576 self.validate_sources_present(current)?;
577 let mut changed = false;
578 for observed in &self.observed {
579 let index = current
580 .binary_search_by_key(&observed.source, |entry| entry.source)
581 .expect("validated availability source remains present");
582 let current_epoch = current[index].epoch;
583 if current_epoch < observed.epoch {
584 return Err(admission_fault(
585 DynamicAdmissionFaultKind::EpochRegression,
586 "capacity availability epoch regressed",
587 ));
588 }
589 changed |= current_epoch > observed.epoch;
590 }
591 Ok(changed)
592 }
593
594 pub(crate) fn refreshed_from(
595 &self,
596 current: &[CapacityAvailabilityEpoch],
597 ) -> Result<Self, VNextError> {
598 self.validate_sources_present(current)?;
599 Self::new(
600 self.coordinator_id,
601 self.observed
602 .iter()
603 .map(|observed| {
604 let index = current
605 .binary_search_by_key(&observed.source, |entry| entry.source)
606 .expect("validated availability source remains present");
607 current[index]
608 })
609 .collect(),
610 )
611 }
612}
613
614#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
621pub struct CapacityWaitSnapshot {
622 epochs: CapacityEpochs,
623 wait_condition: CapacityWaitCondition,
624}
625
626impl CapacityWaitSnapshot {
627 fn new(epochs: CapacityEpochs, wait_condition: CapacityWaitCondition) -> Self {
628 debug_assert_eq!(epochs.coordinator_id(), wait_condition.coordinator_id());
629 Self {
630 epochs,
631 wait_condition,
632 }
633 }
634
635 pub const fn epochs(&self) -> CapacityEpochs {
636 self.epochs
637 }
638
639 pub fn wait_condition(&self) -> &CapacityWaitCondition {
640 &self.wait_condition
641 }
642
643 pub(crate) fn narrow_to_domains(
644 self,
645 domains: impl IntoIterator<Item = CapacityDomainId>,
646 ) -> Result<Self, VNextError> {
647 let sources = domains
648 .into_iter()
649 .map(CapacityAvailabilitySource::Domain)
650 .collect::<BTreeSet<_>>();
651 if sources.is_empty() {
652 return Err(invalid_admission(
653 "capacity wait snapshot cannot be narrowed to no domains",
654 ));
655 }
656 let observed = sources
657 .into_iter()
658 .map(|source| {
659 self.wait_condition
660 .observed
661 .binary_search_by_key(&source, |entry| entry.source)
662 .map(|index| self.wait_condition.observed[index])
663 .map_err(|_| {
664 invalid_admission("capacity wait snapshot cannot add an unobserved domain")
665 })
666 })
667 .collect::<Result<Vec<_>, _>>()?;
668 Ok(Self::new(
669 self.epochs,
670 CapacityWaitCondition::new(self.wait_condition.coordinator_id, observed)?,
671 ))
672 }
673}
674
675#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
676pub struct SequenceAuthorityId {
677 sparse_id: u32,
678 generation: u64,
679}
680
681#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
682pub struct RequestAuthorityId {
683 sparse_id: u32,
684 generation: u64,
685}
686
687impl RequestAuthorityId {
688 #[cfg(test)]
689 pub(crate) const fn test_only(sparse_id: u32, generation: u64) -> Self {
690 Self {
691 sparse_id,
692 generation,
693 }
694 }
695
696 pub const fn sparse_id(self) -> u32 {
697 self.sparse_id
698 }
699
700 pub const fn generation(self) -> u64 {
701 self.generation
702 }
703}
704
705impl SequenceAuthorityId {
706 #[cfg(test)]
707 pub(crate) const fn test_only(sparse_id: u32, generation: u64) -> Self {
708 Self {
709 sparse_id,
710 generation,
711 }
712 }
713
714 pub const fn sparse_id(self) -> u32 {
715 self.sparse_id
716 }
717
718 pub const fn generation(self) -> u64 {
719 self.generation
720 }
721}
722
723#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
724#[serde(rename_all = "snake_case")]
725pub enum DeferredAction {
726 WaitForRelease,
727 AwaitBackingGrowth,
728 PreemptAndRecompute,
729}
730
731#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
732pub struct AdmissionDeferred {
733 immediate_requested: CapacityVector,
734 fit_requested: CapacityVector,
735 available: CapacitySnapshot,
736 blockers: Vec<CapacityShortfall>,
737 action: DeferredAction,
738 release_epoch: u64,
739 capacity_epoch: u64,
740 wait_condition: CapacityWaitCondition,
741}
742
743impl AdmissionDeferred {
744 pub fn immediate_requested(&self) -> &CapacityVector {
745 &self.immediate_requested
746 }
747
748 pub fn fit_requested(&self) -> &CapacityVector {
749 &self.fit_requested
750 }
751
752 pub fn available(&self) -> &CapacitySnapshot {
753 &self.available
754 }
755
756 pub fn blockers(&self) -> &[CapacityShortfall] {
757 &self.blockers
758 }
759
760 pub const fn action(&self) -> DeferredAction {
761 self.action
762 }
763
764 pub const fn release_epoch(&self) -> u64 {
765 self.release_epoch
766 }
767
768 pub const fn capacity_epoch(&self) -> u64 {
769 self.capacity_epoch
770 }
771
772 pub const fn epochs(&self) -> CapacityEpochs {
773 CapacityEpochs {
774 coordinator_id: self.available.coordinator_id,
775 release_epoch: self.release_epoch,
776 capacity_epoch: self.capacity_epoch,
777 }
778 }
779
780 pub fn wait_condition(&self) -> &CapacityWaitCondition {
781 &self.wait_condition
782 }
783}
784
785#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
786pub struct AdmissionRejected {
787 immediate_requested: CapacityVector,
788 fit_requested: CapacityVector,
789 maximum: CapacitySnapshot,
790 blockers: Vec<CapacityShortfall>,
791}
792
793impl AdmissionRejected {
794 pub fn blockers(&self) -> &[CapacityShortfall] {
795 &self.blockers
796 }
797
798 pub fn maximum(&self) -> &CapacitySnapshot {
799 &self.maximum
800 }
801}
802
803#[derive(Debug)]
804pub enum AdmissionDecision {
805 Admitted(LogicalAdmissionLease),
806 Deferred(AdmissionDeferred),
807 PermanentRejected(AdmissionRejected),
808}
809
810#[derive(Debug)]
811pub enum RequestAdmissionDecision {
812 Admitted(LogicalRequestLease),
813 Deferred(AdmissionDeferred),
814 PermanentRejected(AdmissionRejected),
815}
816
817pub(crate) enum InitialSequenceAdmissionDecision {
818 Admitted(LogicalInitialSequenceAdmission),
819 Deferred,
820 PermanentRejected(AdmissionRejected),
821}
822
823pub(crate) struct LogicalInitialSequenceAdmission {
827 sequence: LogicalAdmissionLease,
828 request: LogicalRequestLease,
829}
830
831impl LogicalInitialSequenceAdmission {
832 pub(crate) fn into_parts(self) -> (LogicalRequestLease, LogicalAdmissionLease) {
833 (self.request, self.sequence)
834 }
835}
836
837pub(crate) enum AdmissionPreflightDecision {
838 Eligible,
839 Deferred(AdmissionDeferred),
840 PermanentRejected(AdmissionRejected),
841}
842
843#[derive(Debug)]
844pub enum CapacityClaimDecision {
845 Claimed(LogicalCapacityLease),
846 Deferred(AdmissionDeferred),
847 PermanentRejected(AdmissionRejected),
848}
849
850#[derive(Debug)]
851pub enum BatchCapacityClaimDecision {
852 Claimed(LogicalBatchCapacityLease),
853 Deferred(AdmissionDeferred),
854 PermanentRejected(AdmissionRejected),
855}
856
857#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
862pub struct SequenceCapacityParent {
863 request: RequestAuthorityId,
864 sequence: SequenceAuthorityId,
865}
866
867impl SequenceCapacityParent {
868 pub const fn request(self) -> RequestAuthorityId {
869 self.request
870 }
871
872 pub const fn sequence(self) -> SequenceAuthorityId {
873 self.sequence
874 }
875}
876
877#[derive(Debug)]
878struct DomainState {
879 spec: CapacityDomainSpec,
880 used: u64,
881 availability_epoch: u64,
882}
883
884#[derive(Debug, Clone, Copy)]
885struct LiveSequenceRecord {
886 generation: u64,
887 request: RequestAuthorityId,
888 active_child_claims: u64,
889}
890
891#[derive(Debug, Clone, Copy)]
892struct LiveRequestRecord {
893 generation: u64,
894 active_sequences: u32,
895}
896
897#[derive(Debug)]
898struct CoordinatorState {
899 domains: BTreeMap<CapacityDomainId, DomainState>,
900 maximum_active_sequences: u32,
901 active_requests: u32,
902 active_sequences: u32,
903 active_child_claims: u64,
904 live_requests: Vec<Option<LiveRequestRecord>>,
905 reusable_request_ids: Vec<u32>,
906 live_sequences: Vec<Option<LiveSequenceRecord>>,
907 reusable_sequence_ids: Vec<u32>,
908 next_request_generation: u64,
909 next_sequence_generation: u64,
910 release_epoch: u64,
911 capacity_epoch: u64,
912 active_sequence_availability_epoch: u64,
913 poisoned: bool,
914}
915
916impl CoordinatorState {
917 fn snapshot(&self, coordinator_id: LogicalAdmissionCoordinatorId) -> CapacitySnapshot {
918 let domains = self
919 .domains
920 .iter()
921 .map(|(domain, state)| {
922 let total = state.spec.total_units.get();
923 DomainCapacitySnapshot {
924 domain: *domain,
925 total: CapacityUnits::new(total),
926 maximum_total: state.spec.maximum_total_units,
927 used: CapacityUnits::new(state.used),
928 available: CapacityUnits::new(total.saturating_sub(state.used)),
929 }
930 })
931 .collect();
932 CapacitySnapshot {
933 coordinator_id,
934 domains,
935 active_requests: self.active_requests,
936 active_sequences: self.active_sequences,
937 active_child_claims: self.active_child_claims,
938 maximum_active_sequences: self.maximum_active_sequences,
939 release_epoch: self.release_epoch,
940 capacity_epoch: self.capacity_epoch,
941 live_sequence_records: if self.poisoned {
942 self.live_sequences
943 .iter()
944 .filter(|record| record.is_some())
945 .count()
946 } else {
947 self.active_sequences as usize
948 },
949 reusable_sequence_ids: self.reusable_sequence_ids.len(),
950 live_request_records: if self.poisoned {
951 self.live_requests
952 .iter()
953 .filter(|record| record.is_some())
954 .count()
955 } else {
956 self.active_requests as usize
957 },
958 reusable_request_ids: self.reusable_request_ids.len(),
959 poisoned: self.poisoned,
960 }
961 }
962
963 fn epochs(&self, coordinator_id: LogicalAdmissionCoordinatorId) -> CapacityEpochs {
964 CapacityEpochs {
965 coordinator_id,
966 release_epoch: self.release_epoch,
967 capacity_epoch: self.capacity_epoch,
968 }
969 }
970
971 fn write_availability_epochs(&self, out: &mut Vec<CapacityAvailabilityEpoch>) {
972 out.clear();
973 out.extend(
974 self.domains
975 .iter()
976 .map(|(domain, state)| CapacityAvailabilityEpoch {
977 source: CapacityAvailabilitySource::Domain(*domain),
978 epoch: state.availability_epoch,
979 }),
980 );
981 out.push(CapacityAvailabilityEpoch {
982 source: CapacityAvailabilitySource::ActiveSequenceSlots,
983 epoch: self.active_sequence_availability_epoch,
984 });
985 }
986
987 fn wait_condition_for_blockers(
988 &self,
989 coordinator_id: LogicalAdmissionCoordinatorId,
990 blockers: &[CapacityShortfall],
991 ) -> Result<CapacityWaitCondition, VNextError> {
992 let mut sources = BTreeSet::new();
993 for blocker in blockers {
994 match (blocker.kind, blocker.domain) {
995 (CapacityShortfallKind::ActiveSequenceCeiling, None) => {
996 sources.insert(CapacityAvailabilitySource::ActiveSequenceSlots);
997 }
998 (CapacityShortfallKind::PermanentDomainMaximum, _) => {
999 return Err(invalid_admission(
1000 "permanent capacity blocker cannot produce a wait condition",
1001 ));
1002 }
1003 (_, Some(domain)) => {
1004 if !self.domains.contains_key(&domain) {
1005 return Err(admission_fault(
1006 DynamicAdmissionFaultKind::UnknownDomain,
1007 "capacity blocker references an unknown wait domain",
1008 ));
1009 }
1010 sources.insert(CapacityAvailabilitySource::Domain(domain));
1011 }
1012 (_, None) => {
1013 return Err(invalid_admission(
1014 "capacity blocker contains no availability source",
1015 ));
1016 }
1017 }
1018 }
1019 self.wait_condition_for_sources(coordinator_id, sources)
1020 }
1021
1022 fn wait_condition_for_domains(
1023 &self,
1024 coordinator_id: LogicalAdmissionCoordinatorId,
1025 domains: impl IntoIterator<Item = CapacityDomainId>,
1026 ) -> Result<CapacityWaitCondition, VNextError> {
1027 self.wait_condition_for_sources(
1028 coordinator_id,
1029 domains
1030 .into_iter()
1031 .map(CapacityAvailabilitySource::Domain)
1032 .collect(),
1033 )
1034 }
1035
1036 fn wait_condition_for_sources(
1037 &self,
1038 coordinator_id: LogicalAdmissionCoordinatorId,
1039 sources: BTreeSet<CapacityAvailabilitySource>,
1040 ) -> Result<CapacityWaitCondition, VNextError> {
1041 let observed = sources
1042 .into_iter()
1043 .map(|source| {
1044 let epoch = match source {
1045 CapacityAvailabilitySource::Domain(domain) => {
1046 self.domains
1047 .get(&domain)
1048 .ok_or_else(|| {
1049 admission_fault(
1050 DynamicAdmissionFaultKind::UnknownDomain,
1051 "capacity wait references an unknown domain",
1052 )
1053 })?
1054 .availability_epoch
1055 }
1056 CapacityAvailabilitySource::ActiveSequenceSlots => {
1057 self.active_sequence_availability_epoch
1058 }
1059 CapacityAvailabilitySource::PlanDeviceBudget
1060 | CapacityAvailabilitySource::ProcessDeviceCapacity => {
1061 return Err(invalid_admission(
1062 "device capacity availability is owned by the device account",
1063 ));
1064 }
1065 };
1066 CapacityAvailabilityEpoch::new(source, epoch)
1067 })
1068 .collect::<Result<Vec<_>, _>>()?;
1069 CapacityWaitCondition::new(coordinator_id, observed)
1070 }
1071
1072 fn availability_epoch_for(
1073 &self,
1074 source: CapacityAvailabilitySource,
1075 ) -> Result<u64, VNextError> {
1076 match source {
1077 CapacityAvailabilitySource::Domain(domain) => self
1078 .domains
1079 .get(&domain)
1080 .map(|state| state.availability_epoch)
1081 .ok_or_else(|| {
1082 admission_fault(
1083 DynamicAdmissionFaultKind::UnknownDomain,
1084 "capacity wait references an unknown domain",
1085 )
1086 }),
1087 CapacityAvailabilitySource::ActiveSequenceSlots => {
1088 Ok(self.active_sequence_availability_epoch)
1089 }
1090 CapacityAvailabilitySource::PlanDeviceBudget
1091 | CapacityAvailabilitySource::ProcessDeviceCapacity => Err(invalid_admission(
1092 "device capacity availability is owned by the device account",
1093 )),
1094 }
1095 }
1096
1097 fn wait_condition_changed(
1098 &self,
1099 coordinator_id: LogicalAdmissionCoordinatorId,
1100 observed: &CapacityWaitCondition,
1101 ) -> Result<bool, VNextError> {
1102 if observed.coordinator_id != coordinator_id {
1103 return Err(admission_fault(
1104 DynamicAdmissionFaultKind::ForeignCoordinator,
1105 "capacity wait condition belongs to a different coordinator",
1106 ));
1107 }
1108 let mut changed = false;
1109 for entry in &observed.observed {
1110 let current = self.availability_epoch_for(entry.source)?;
1111 if current < entry.epoch {
1112 return Err(admission_fault(
1113 DynamicAdmissionFaultKind::EpochRegression,
1114 "capacity availability epoch regressed",
1115 ));
1116 }
1117 changed |= current > entry.epoch;
1118 }
1119 Ok(changed)
1120 }
1121
1122 fn refresh_wait_condition(
1123 &self,
1124 coordinator_id: LogicalAdmissionCoordinatorId,
1125 observed: &CapacityWaitCondition,
1126 ) -> Result<CapacityWaitCondition, VNextError> {
1127 if observed.coordinator_id != coordinator_id {
1128 return Err(admission_fault(
1129 DynamicAdmissionFaultKind::ForeignCoordinator,
1130 "capacity wait condition belongs to a different coordinator",
1131 ));
1132 }
1133 CapacityWaitCondition::new(
1134 coordinator_id,
1135 observed
1136 .observed
1137 .iter()
1138 .map(|entry| {
1139 CapacityAvailabilityEpoch::new(
1140 entry.source,
1141 self.availability_epoch_for(entry.source)?,
1142 )
1143 })
1144 .collect::<Result<Vec<_>, _>>()?,
1145 )
1146 }
1147
1148 fn preview_request_authority(&self) -> Result<RequestAuthorityReservation, VNextError> {
1149 let generation = self.next_request_generation;
1150 let next_generation = generation.checked_add(1).ok_or_else(|| {
1151 admission_fault(
1152 DynamicAdmissionFaultKind::AuthorityExhausted,
1153 "request authority generation is exhausted",
1154 )
1155 })?;
1156 let (sparse_id, source) = if let Some(id) = self.reusable_request_ids.last().copied() {
1157 (id, SparseIdSource::Reused)
1158 } else {
1159 if self.live_requests.len() >= u32::MAX as usize {
1160 return Err(admission_fault(
1161 DynamicAdmissionFaultKind::AuthorityExhausted,
1162 "request authority id space is exhausted",
1163 ));
1164 }
1165 (self.live_requests.len() as u32, SparseIdSource::Fresh)
1166 };
1167 if self
1168 .live_requests
1169 .get(sparse_id as usize)
1170 .is_some_and(Option::is_some)
1171 {
1172 return Err(invalid_admission(
1173 "request authority allocator selected a live sparse id",
1174 ));
1175 }
1176 Ok(RequestAuthorityReservation {
1177 authority: RequestAuthorityId {
1178 sparse_id,
1179 generation,
1180 },
1181 source,
1182 next_generation,
1183 })
1184 }
1185
1186 fn prepare_request_storage(
1187 &mut self,
1188 reservation: RequestAuthorityReservation,
1189 ) -> Result<(), VNextError> {
1190 if matches!(reservation.source, SparseIdSource::Fresh) {
1191 self.live_requests.try_reserve(1).map_err(|_| {
1192 admission_fault(
1193 DynamicAdmissionFaultKind::AllocationFailure,
1194 "cannot reserve request authority slab",
1195 )
1196 })?;
1197 self.reusable_request_ids
1198 .try_reserve(self.live_requests.len() + 1)
1199 .map_err(|_| {
1200 admission_fault(
1201 DynamicAdmissionFaultKind::AllocationFailure,
1202 "cannot reserve request release free-list",
1203 )
1204 })?;
1205 }
1206 Ok(())
1207 }
1208
1209 fn commit_request_authority(
1210 &mut self,
1211 reservation: RequestAuthorityReservation,
1212 ) -> RequestAuthorityId {
1213 let record = LiveRequestRecord {
1214 generation: reservation.authority.generation,
1215 active_sequences: 0,
1216 };
1217 match reservation.source {
1218 SparseIdSource::Reused => {
1219 self.reusable_request_ids.pop();
1220 self.live_requests[reservation.authority.sparse_id as usize] = Some(record);
1221 }
1222 SparseIdSource::Fresh => self.live_requests.push(Some(record)),
1223 }
1224 self.next_request_generation = reservation.next_generation;
1225 reservation.authority
1226 }
1227
1228 fn preview_sequence_authority(&self) -> Result<SequenceAuthorityReservation, VNextError> {
1229 let generation = self.next_sequence_generation;
1230 let next_generation = generation.checked_add(1).ok_or_else(|| {
1231 admission_fault(
1232 DynamicAdmissionFaultKind::AuthorityExhausted,
1233 "sequence authority generation is exhausted",
1234 )
1235 })?;
1236 let (sparse_id, source) = if let Some(id) = self.reusable_sequence_ids.last().copied() {
1237 (id, SparseIdSource::Reused)
1238 } else {
1239 if self.live_sequences.len() >= u32::MAX as usize {
1240 return Err(admission_fault(
1241 DynamicAdmissionFaultKind::AuthorityExhausted,
1242 "sequence authority id space is exhausted",
1243 ));
1244 }
1245 (self.live_sequences.len() as u32, SparseIdSource::Fresh)
1246 };
1247 if self
1248 .live_sequences
1249 .get(sparse_id as usize)
1250 .is_some_and(Option::is_some)
1251 {
1252 return Err(invalid_admission(
1253 "sequence authority allocator selected a live sparse id",
1254 ));
1255 }
1256 Ok(SequenceAuthorityReservation {
1257 authority: SequenceAuthorityId {
1258 sparse_id,
1259 generation,
1260 },
1261 source,
1262 next_generation,
1263 })
1264 }
1265
1266 fn prepare_sequence_storage(
1267 &mut self,
1268 reservation: SequenceAuthorityReservation,
1269 ) -> Result<(), VNextError> {
1270 if matches!(reservation.source, SparseIdSource::Fresh) {
1271 self.live_sequences.try_reserve(1).map_err(|_| {
1272 admission_fault(
1273 DynamicAdmissionFaultKind::AllocationFailure,
1274 "cannot reserve sequence authority slab",
1275 )
1276 })?;
1277 self.reusable_sequence_ids
1278 .try_reserve(self.live_sequences.len() + 1)
1279 .map_err(|_| {
1280 admission_fault(
1281 DynamicAdmissionFaultKind::AllocationFailure,
1282 "cannot reserve sequence release free-list",
1283 )
1284 })?;
1285 }
1286 Ok(())
1287 }
1288
1289 fn commit_sequence_authority(
1290 &mut self,
1291 reservation: SequenceAuthorityReservation,
1292 request: RequestAuthorityId,
1293 ) -> SequenceAuthorityId {
1294 match reservation.source {
1295 SparseIdSource::Reused => {
1296 self.reusable_sequence_ids.pop();
1297 self.live_sequences[reservation.authority.sparse_id as usize] =
1298 Some(LiveSequenceRecord {
1299 generation: reservation.authority.generation,
1300 request,
1301 active_child_claims: 0,
1302 });
1303 }
1304 SparseIdSource::Fresh => {
1305 self.live_sequences.push(Some(LiveSequenceRecord {
1306 generation: reservation.authority.generation,
1307 request,
1308 active_child_claims: 0,
1309 }));
1310 }
1311 }
1312 self.next_sequence_generation = reservation.next_generation;
1313 reservation.authority
1314 }
1315}
1316
1317#[derive(Debug, Clone, Copy)]
1318enum SparseIdSource {
1319 Reused,
1320 Fresh,
1321}
1322
1323#[derive(Debug, Clone, Copy)]
1324struct RequestAuthorityReservation {
1325 authority: RequestAuthorityId,
1326 source: SparseIdSource,
1327 next_generation: u64,
1328}
1329
1330#[derive(Debug, Clone, Copy)]
1331struct SequenceAuthorityReservation {
1332 authority: SequenceAuthorityId,
1333 source: SparseIdSource,
1334 next_generation: u64,
1335}
1336
1337#[derive(Debug)]
1338struct CoordinatorInner {
1339 id: LogicalAdmissionCoordinatorId,
1340 state: Mutex<CoordinatorState>,
1341 epoch_tx: watch::Sender<CapacityEpochs>,
1342}
1343
1344impl CoordinatorInner {
1345 fn lock_state(&self) -> Result<MutexGuard<'_, CoordinatorState>, VNextError> {
1346 match self.state.lock() {
1347 Ok(state) => Ok(state),
1348 Err(poisoned) => {
1349 let mut state = poisoned.into_inner();
1350 state.poisoned = true;
1351 let epochs = state.epochs(self.id);
1352 self.epoch_tx.send_replace(epochs);
1353 Err(admission_fault(
1354 DynamicAdmissionFaultKind::Poisoned,
1355 "coordinator state is poisoned",
1356 ))
1357 }
1358 }
1359 }
1360
1361 fn lock_mutation(&self) -> Result<CoordinatorMutationGuard<'_>, VNextError> {
1362 Ok(CoordinatorMutationGuard {
1363 inner: self,
1364 state: self.lock_state()?,
1365 panicking_on_entry: std::thread::panicking(),
1366 })
1367 }
1368}
1369
1370struct CoordinatorMutationGuard<'a> {
1371 inner: &'a CoordinatorInner,
1372 state: MutexGuard<'a, CoordinatorState>,
1373 panicking_on_entry: bool,
1374}
1375
1376impl Deref for CoordinatorMutationGuard<'_> {
1377 type Target = CoordinatorState;
1378
1379 fn deref(&self) -> &Self::Target {
1380 &self.state
1381 }
1382}
1383
1384impl DerefMut for CoordinatorMutationGuard<'_> {
1385 fn deref_mut(&mut self) -> &mut Self::Target {
1386 &mut self.state
1387 }
1388}
1389
1390impl Drop for CoordinatorMutationGuard<'_> {
1391 fn drop(&mut self) {
1392 if !self.panicking_on_entry && std::thread::panicking() {
1393 self.state.poisoned = true;
1394 self.inner
1395 .epoch_tx
1396 .send_replace(self.state.epochs(self.inner.id));
1397 }
1398 }
1399}
1400
1401#[derive(Debug, Clone)]
1402pub struct LogicalAdmissionCoordinator {
1403 inner: Arc<CoordinatorInner>,
1404}
1405
1406impl LogicalAdmissionCoordinator {
1407 pub(crate) fn new(
1408 domains: Vec<(CapacityDomainId, CapacityDomainSpec)>,
1409 maximum_active_sequences: u32,
1410 ) -> Result<Self, VNextError> {
1411 if maximum_active_sequences == 0 {
1412 return Err(invalid_admission(
1413 "coordinator requires a non-zero sequence ceiling",
1414 ));
1415 }
1416 let mut registered = BTreeMap::new();
1417 for (domain, spec) in domains {
1418 if registered
1419 .insert(
1420 domain,
1421 DomainState {
1422 spec,
1423 used: 0,
1424 availability_epoch: 1,
1425 },
1426 )
1427 .is_some()
1428 {
1429 return Err(invalid_admission("duplicate coordinator capacity domain"));
1430 }
1431 }
1432 let coordinator_id = LogicalAdmissionCoordinatorId::issue()?;
1433 let initial_epochs = CapacityEpochs {
1434 coordinator_id,
1435 release_epoch: 1,
1436 capacity_epoch: 1,
1437 };
1438 let (epoch_tx, _) = watch::channel(initial_epochs);
1439 Ok(Self {
1440 inner: Arc::new(CoordinatorInner {
1441 id: coordinator_id,
1442 state: Mutex::new(CoordinatorState {
1443 domains: registered,
1444 maximum_active_sequences,
1445 active_requests: 0,
1446 active_sequences: 0,
1447 active_child_claims: 0,
1448 live_requests: Vec::new(),
1449 reusable_request_ids: Vec::new(),
1450 live_sequences: Vec::new(),
1451 reusable_sequence_ids: Vec::new(),
1452 next_request_generation: 1,
1453 next_sequence_generation: 1,
1454 release_epoch: 1,
1455 capacity_epoch: 1,
1456 active_sequence_availability_epoch: 1,
1457 poisoned: false,
1458 }),
1459 epoch_tx,
1460 }),
1461 })
1462 }
1463
1464 pub fn id(&self) -> LogicalAdmissionCoordinatorId {
1465 self.inner.id
1466 }
1467
1468 pub(crate) fn owns(&self, lease: &LogicalAdmissionLease) -> bool {
1469 self.id() == lease.coordinator_id() && Arc::ptr_eq(&self.inner, &lease.inner)
1470 }
1471
1472 pub(crate) fn owns_request(&self, lease: &LogicalRequestLease) -> bool {
1473 self.id() == lease.coordinator_id() && Arc::ptr_eq(&self.inner, &lease.inner)
1474 }
1475
1476 pub(crate) fn try_admit_request(
1477 &self,
1478 demand: &AdmissionDemand,
1479 ) -> Result<RequestAdmissionDecision, VNextError> {
1480 let mut state = self.inner.lock_mutation()?;
1481 if state.poisoned {
1482 return Err(admission_fault(
1483 DynamicAdmissionFaultKind::Poisoned,
1484 "coordinator is fail-closed",
1485 ));
1486 }
1487
1488 let evaluation = evaluate_demand(&state, demand)?;
1489 if !evaluation.permanent.is_empty() {
1490 return Ok(RequestAdmissionDecision::PermanentRejected(
1491 AdmissionRejected {
1492 immediate_requested: demand.immediate_claim.clone(),
1493 fit_requested: demand.fit_requirement.clone(),
1494 maximum: state.snapshot(self.id()),
1495 blockers: evaluation.permanent,
1496 },
1497 ));
1498 }
1499 if !evaluation.blockers.is_empty() {
1500 let action = deferred_action(demand, evaluation.growth_required);
1501 let wait_condition =
1502 state.wait_condition_for_blockers(self.id(), &evaluation.blockers)?;
1503 let snapshot = state.snapshot(self.id());
1504 return Ok(RequestAdmissionDecision::Deferred(AdmissionDeferred {
1505 immediate_requested: demand.immediate_claim.clone(),
1506 fit_requested: demand.fit_requirement.clone(),
1507 release_epoch: snapshot.release_epoch,
1508 capacity_epoch: snapshot.capacity_epoch,
1509 available: snapshot,
1510 blockers: evaluation.blockers,
1511 action,
1512 wait_condition,
1513 }));
1514 }
1515
1516 let request = state.preview_request_authority()?;
1517 let committed_claims = demand.immediate_claim.clone();
1518 let mut next_usage = Vec::with_capacity(demand.immediate_claim.entries().len());
1519 for entry in demand.immediate_claim.entries() {
1520 let domain = state
1521 .domains
1522 .get(&entry.domain)
1523 .expect("known request demand domain was preflight validated");
1524 let used = domain.used.checked_add(entry.units.get()).ok_or_else(|| {
1525 admission_fault(
1526 DynamicAdmissionFaultKind::ArithmeticOverflow,
1527 "request capacity usage overflows u64",
1528 )
1529 })?;
1530 next_usage.push((entry.domain, used));
1531 }
1532 let next_active_requests = state.active_requests.checked_add(1).ok_or_else(|| {
1533 admission_fault(
1534 DynamicAdmissionFaultKind::AuthorityExhausted,
1535 "active request count is exhausted",
1536 )
1537 })?;
1538 state
1539 .release_epoch
1540 .checked_add(u64::from(next_active_requests))
1541 .and_then(|epoch| epoch.checked_add(u64::from(state.active_sequences)))
1542 .and_then(|epoch| epoch.checked_add(state.active_child_claims))
1543 .ok_or_else(|| {
1544 admission_fault(
1545 DynamicAdmissionFaultKind::EpochExhausted,
1546 "release epoch cannot represent every outstanding lease release",
1547 )
1548 })?;
1549 state.prepare_request_storage(request)?;
1550 let request = state.commit_request_authority(request);
1551 for (domain, used) in next_usage {
1552 state
1553 .domains
1554 .get_mut(&domain)
1555 .expect("validated request capacity domain remains registered")
1556 .used = used;
1557 }
1558 state.active_requests = next_active_requests;
1559 Ok(RequestAdmissionDecision::Admitted(LogicalRequestLease {
1560 inner: Arc::clone(&self.inner),
1561 request,
1562 claims: committed_claims,
1563 released: false,
1564 }))
1565 }
1566
1567 pub(crate) fn preflight_initial_sequence(
1573 &self,
1574 request: &AdmissionDemand,
1575 sequence: &AdmissionDemand,
1576 ) -> Result<AdmissionPreflightDecision, VNextError> {
1577 let demand = AdmissionDemand::initial_sequence_bundle(request, sequence)?;
1578 let state = self.inner.lock_state()?;
1579 if state.poisoned {
1580 return Err(admission_fault(
1581 DynamicAdmissionFaultKind::Poisoned,
1582 "coordinator is fail-closed",
1583 ));
1584 }
1585 let mut evaluation = evaluate_demand(&state, &demand)?;
1586 if !evaluation.permanent.is_empty() {
1587 return Ok(AdmissionPreflightDecision::PermanentRejected(
1588 AdmissionRejected {
1589 immediate_requested: demand.immediate_claim,
1590 fit_requested: demand.fit_requirement,
1591 maximum: state.snapshot(self.id()),
1592 blockers: evaluation.permanent,
1593 },
1594 ));
1595 }
1596 if state.active_sequences < state.maximum_active_sequences {
1597 return Ok(AdmissionPreflightDecision::Eligible);
1598 }
1599 evaluation.blockers.push(CapacityShortfall {
1600 domain: None,
1601 kind: CapacityShortfallKind::ActiveSequenceCeiling,
1602 requested: CapacityUnits::new(1),
1603 available: CapacityUnits::ZERO,
1604 current_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
1605 maximum_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
1606 });
1607 let wait_condition = state.wait_condition_for_blockers(self.id(), &evaluation.blockers)?;
1608 let snapshot = state.snapshot(self.id());
1609 Ok(AdmissionPreflightDecision::Deferred(AdmissionDeferred {
1610 immediate_requested: demand.immediate_claim,
1611 fit_requested: demand.fit_requirement,
1612 release_epoch: snapshot.release_epoch,
1613 capacity_epoch: snapshot.capacity_epoch,
1614 available: snapshot,
1615 blockers: evaluation.blockers,
1616 action: match demand.pressure_action {
1617 AdmissionPressureAction::WaitForRelease => DeferredAction::WaitForRelease,
1618 AdmissionPressureAction::PreemptAndRecompute => DeferredAction::PreemptAndRecompute,
1619 },
1620 wait_condition,
1621 }))
1622 }
1623
1624 pub(crate) fn observe_initial_sequence(
1628 &self,
1629 request: &AdmissionDemand,
1630 sequence: &AdmissionDemand,
1631 ) -> Result<AdmissionPreflightDecision, VNextError> {
1632 let demand = AdmissionDemand::initial_sequence_bundle(request, sequence)?;
1633 let state = self.inner.lock_state()?;
1634 if state.poisoned {
1635 return Err(admission_fault(
1636 DynamicAdmissionFaultKind::Poisoned,
1637 "coordinator is fail-closed",
1638 ));
1639 }
1640 let mut evaluation = evaluate_demand(&state, &demand)?;
1641 if !evaluation.permanent.is_empty() {
1642 return Ok(AdmissionPreflightDecision::PermanentRejected(
1643 AdmissionRejected {
1644 immediate_requested: demand.immediate_claim,
1645 fit_requested: demand.fit_requirement,
1646 maximum: state.snapshot(self.id()),
1647 blockers: evaluation.permanent,
1648 },
1649 ));
1650 }
1651 if state.active_sequences >= state.maximum_active_sequences {
1652 evaluation.blockers.push(CapacityShortfall {
1653 domain: None,
1654 kind: CapacityShortfallKind::ActiveSequenceCeiling,
1655 requested: CapacityUnits::new(1),
1656 available: CapacityUnits::ZERO,
1657 current_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
1658 maximum_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
1659 });
1660 }
1661 if evaluation.blockers.is_empty() {
1662 return Ok(AdmissionPreflightDecision::Eligible);
1663 }
1664 let action = deferred_action(&demand, evaluation.growth_required);
1665 let wait_condition = state.wait_condition_for_blockers(self.id(), &evaluation.blockers)?;
1666 let snapshot = state.snapshot(self.id());
1667 Ok(AdmissionPreflightDecision::Deferred(AdmissionDeferred {
1668 immediate_requested: demand.immediate_claim,
1669 fit_requested: demand.fit_requirement,
1670 release_epoch: snapshot.release_epoch,
1671 capacity_epoch: snapshot.capacity_epoch,
1672 available: snapshot,
1673 blockers: evaluation.blockers,
1674 action,
1675 wait_condition,
1676 }))
1677 }
1678
1679 pub(crate) fn try_admit_initial_sequence(
1683 &self,
1684 request_demand: &AdmissionDemand,
1685 sequence_demand: &AdmissionDemand,
1686 ) -> Result<InitialSequenceAdmissionDecision, VNextError> {
1687 let demand = AdmissionDemand::initial_sequence_bundle(request_demand, sequence_demand)?;
1688 let mut state = self.inner.lock_mutation()?;
1689 if state.poisoned {
1690 return Err(admission_fault(
1691 DynamicAdmissionFaultKind::Poisoned,
1692 "coordinator is fail-closed",
1693 ));
1694 }
1695
1696 let mut evaluation = evaluate_demand(&state, &demand)?;
1697 if !evaluation.permanent.is_empty() {
1698 return Ok(InitialSequenceAdmissionDecision::PermanentRejected(
1699 AdmissionRejected {
1700 immediate_requested: demand.immediate_claim,
1701 fit_requested: demand.fit_requirement,
1702 maximum: state.snapshot(self.id()),
1703 blockers: evaluation.permanent,
1704 },
1705 ));
1706 }
1707 if state.active_sequences >= state.maximum_active_sequences {
1708 evaluation.blockers.push(CapacityShortfall {
1709 domain: None,
1710 kind: CapacityShortfallKind::ActiveSequenceCeiling,
1711 requested: CapacityUnits::new(1),
1712 available: CapacityUnits::ZERO,
1713 current_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
1714 maximum_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
1715 });
1716 }
1717 if !evaluation.blockers.is_empty() {
1718 return Ok(InitialSequenceAdmissionDecision::Deferred);
1719 }
1720
1721 let request_reservation = state.preview_request_authority()?;
1722 let sequence_reservation = state.preview_sequence_authority()?;
1723 let mut next_usage = Vec::with_capacity(demand.immediate_claim.entries().len());
1724 for entry in demand.immediate_claim.entries() {
1725 let domain = state
1726 .domains
1727 .get(&entry.domain)
1728 .expect("known bundle demand domain was preflight validated");
1729 let used = domain.used.checked_add(entry.units.get()).ok_or_else(|| {
1730 admission_fault(
1731 DynamicAdmissionFaultKind::ArithmeticOverflow,
1732 "initial request/sequence capacity usage overflows u64",
1733 )
1734 })?;
1735 next_usage.push((entry.domain, used));
1736 }
1737 let next_active_requests = state.active_requests.checked_add(1).ok_or_else(|| {
1738 admission_fault(
1739 DynamicAdmissionFaultKind::AuthorityExhausted,
1740 "active request count is exhausted",
1741 )
1742 })?;
1743 let next_active_sequences = state.active_sequences.checked_add(1).ok_or_else(|| {
1744 admission_fault(
1745 DynamicAdmissionFaultKind::ArithmeticOverflow,
1746 "active sequence count overflows u32",
1747 )
1748 })?;
1749 state
1750 .release_epoch
1751 .checked_add(u64::from(next_active_requests))
1752 .and_then(|epoch| epoch.checked_add(u64::from(next_active_sequences)))
1753 .and_then(|epoch| epoch.checked_add(state.active_child_claims))
1754 .ok_or_else(|| {
1755 admission_fault(
1756 DynamicAdmissionFaultKind::EpochExhausted,
1757 "release epoch cannot represent the initial request/sequence bundle",
1758 )
1759 })?;
1760 state.prepare_request_storage(request_reservation)?;
1761 state.prepare_sequence_storage(sequence_reservation)?;
1762
1763 let request = state.commit_request_authority(request_reservation);
1764 let sequence = state.commit_sequence_authority(sequence_reservation, request);
1765 for (domain, used) in next_usage {
1766 state
1767 .domains
1768 .get_mut(&domain)
1769 .expect("validated bundle capacity domain remains registered")
1770 .used = used;
1771 }
1772 state.active_requests = next_active_requests;
1773 state.active_sequences = next_active_sequences;
1774 state.live_requests[request.sparse_id as usize]
1775 .as_mut()
1776 .expect("new initial request remains live")
1777 .active_sequences = 1;
1778
1779 Ok(InitialSequenceAdmissionDecision::Admitted(
1780 LogicalInitialSequenceAdmission {
1781 sequence: LogicalAdmissionLease {
1782 inner: Arc::clone(&self.inner),
1783 request,
1784 sequence,
1785 claims: sequence_demand.immediate_claim.clone(),
1786 released: false,
1787 },
1788 request: LogicalRequestLease {
1789 inner: Arc::clone(&self.inner),
1790 request,
1791 claims: request_demand.immediate_claim.clone(),
1792 released: false,
1793 },
1794 },
1795 ))
1796 }
1797
1798 pub(crate) fn try_admit_sequence_for_request(
1799 &self,
1800 request: &LogicalRequestLease,
1801 demand: &AdmissionDemand,
1802 ) -> Result<AdmissionDecision, VNextError> {
1803 if !self.owns_request(request) {
1804 return Err(admission_fault(
1805 DynamicAdmissionFaultKind::ForeignCoordinator,
1806 "parent request belongs to another coordinator",
1807 ));
1808 }
1809 if request.released {
1810 return Err(invalid_admission(
1811 "sequence admission requires a live parent request",
1812 ));
1813 }
1814 let mut state = self.inner.lock_mutation()?;
1815 if state.poisoned {
1816 return Err(admission_fault(
1817 DynamicAdmissionFaultKind::Poisoned,
1818 "coordinator is fail-closed",
1819 ));
1820 }
1821 let request_record = state
1822 .live_requests
1823 .get(request.request.sparse_id as usize)
1824 .and_then(|record| *record)
1825 .filter(|record| record.generation == request.request.generation)
1826 .ok_or_else(|| invalid_admission("parent request authority is stale or not live"))?;
1827
1828 let mut evaluation = evaluate_demand(&state, demand)?;
1829 if !evaluation.permanent.is_empty() {
1830 return Ok(AdmissionDecision::PermanentRejected(AdmissionRejected {
1831 immediate_requested: demand.immediate_claim.clone(),
1832 fit_requested: demand.fit_requirement.clone(),
1833 maximum: state.snapshot(self.id()),
1834 blockers: evaluation.permanent,
1835 }));
1836 }
1837 if state.active_sequences >= state.maximum_active_sequences {
1838 evaluation.blockers.push(CapacityShortfall {
1839 domain: None,
1840 kind: CapacityShortfallKind::ActiveSequenceCeiling,
1841 requested: CapacityUnits::new(1),
1842 available: CapacityUnits::ZERO,
1843 current_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
1844 maximum_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
1845 });
1846 }
1847 if !evaluation.blockers.is_empty() {
1848 let action = deferred_action(demand, evaluation.growth_required);
1849 let wait_condition =
1850 state.wait_condition_for_blockers(self.id(), &evaluation.blockers)?;
1851 let snapshot = state.snapshot(self.id());
1852 return Ok(AdmissionDecision::Deferred(AdmissionDeferred {
1853 immediate_requested: demand.immediate_claim.clone(),
1854 fit_requested: demand.fit_requirement.clone(),
1855 release_epoch: snapshot.release_epoch,
1856 capacity_epoch: snapshot.capacity_epoch,
1857 available: snapshot,
1858 blockers: evaluation.blockers,
1859 action,
1860 wait_condition,
1861 }));
1862 }
1863
1864 let sequence = state.preview_sequence_authority()?;
1865 let committed_claims = demand.immediate_claim.clone();
1866 let mut next_usage = Vec::with_capacity(demand.immediate_claim.entries().len());
1867 for entry in demand.immediate_claim.entries() {
1868 let domain = state
1869 .domains
1870 .get(&entry.domain)
1871 .expect("known demand domain was preflight validated");
1872 let used = domain.used.checked_add(entry.units.get()).ok_or_else(|| {
1873 admission_fault(
1874 DynamicAdmissionFaultKind::ArithmeticOverflow,
1875 "capacity usage overflows u64",
1876 )
1877 })?;
1878 next_usage.push((entry.domain, used));
1879 }
1880 let next_active_sequences = state.active_sequences.checked_add(1).ok_or_else(|| {
1881 admission_fault(
1882 DynamicAdmissionFaultKind::ArithmeticOverflow,
1883 "active sequence count overflows u32",
1884 )
1885 })?;
1886 let next_request_sequences =
1887 request_record
1888 .active_sequences
1889 .checked_add(1)
1890 .ok_or_else(|| {
1891 admission_fault(
1892 DynamicAdmissionFaultKind::AuthorityExhausted,
1893 "request child sequence count is exhausted",
1894 )
1895 })?;
1896 state
1897 .release_epoch
1898 .checked_add(u64::from(state.active_requests))
1899 .and_then(|epoch| epoch.checked_add(u64::from(next_active_sequences)))
1900 .and_then(|epoch| epoch.checked_add(state.active_child_claims))
1901 .ok_or_else(|| {
1902 admission_fault(
1903 DynamicAdmissionFaultKind::EpochExhausted,
1904 "release epoch cannot represent every outstanding lease release",
1905 )
1906 })?;
1907 state.prepare_sequence_storage(sequence)?;
1908 let sequence = state.commit_sequence_authority(sequence, request.request);
1909 for (domain, used) in next_usage {
1910 state
1911 .domains
1912 .get_mut(&domain)
1913 .expect("validated capacity domain remains registered")
1914 .used = used;
1915 }
1916 state.active_sequences = next_active_sequences;
1917 state.live_requests[request.request.sparse_id as usize]
1918 .as_mut()
1919 .expect("validated parent request remains live")
1920 .active_sequences = next_request_sequences;
1921 Ok(AdmissionDecision::Admitted(LogicalAdmissionLease {
1922 inner: Arc::clone(&self.inner),
1923 request: request.request,
1924 sequence,
1925 claims: committed_claims,
1926 released: false,
1927 }))
1928 }
1929
1930 pub(crate) fn preflight_sequence_ceiling_for_request(
1935 &self,
1936 request: &LogicalRequestLease,
1937 demand: &AdmissionDemand,
1938 ) -> Result<AdmissionPreflightDecision, VNextError> {
1939 if !self.owns_request(request) {
1940 return Err(admission_fault(
1941 DynamicAdmissionFaultKind::ForeignCoordinator,
1942 "parent request belongs to another coordinator",
1943 ));
1944 }
1945 if request.released {
1946 return Err(invalid_admission(
1947 "sequence admission requires a live parent request",
1948 ));
1949 }
1950 let state = self.inner.lock_state()?;
1951 if state.poisoned {
1952 return Err(admission_fault(
1953 DynamicAdmissionFaultKind::Poisoned,
1954 "coordinator is fail-closed",
1955 ));
1956 }
1957 state
1958 .live_requests
1959 .get(request.request.sparse_id as usize)
1960 .and_then(|record| *record)
1961 .filter(|record| record.generation == request.request.generation)
1962 .ok_or_else(|| invalid_admission("parent request authority is stale or not live"))?;
1963
1964 let mut evaluation = evaluate_demand(&state, demand)?;
1965 if !evaluation.permanent.is_empty() {
1966 return Ok(AdmissionPreflightDecision::PermanentRejected(
1967 AdmissionRejected {
1968 immediate_requested: demand.immediate_claim.clone(),
1969 fit_requested: demand.fit_requirement.clone(),
1970 maximum: state.snapshot(self.id()),
1971 blockers: evaluation.permanent,
1972 },
1973 ));
1974 }
1975 if state.active_sequences < state.maximum_active_sequences {
1976 return Ok(AdmissionPreflightDecision::Eligible);
1977 }
1978 evaluation.blockers.push(CapacityShortfall {
1979 domain: None,
1980 kind: CapacityShortfallKind::ActiveSequenceCeiling,
1981 requested: CapacityUnits::new(1),
1982 available: CapacityUnits::ZERO,
1983 current_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
1984 maximum_total: CapacityUnits::new(u64::from(state.maximum_active_sequences)),
1985 });
1986 let wait_condition = state.wait_condition_for_blockers(self.id(), &evaluation.blockers)?;
1987 let snapshot = state.snapshot(self.id());
1988 Ok(AdmissionPreflightDecision::Deferred(AdmissionDeferred {
1989 immediate_requested: demand.immediate_claim.clone(),
1990 fit_requested: demand.fit_requirement.clone(),
1991 release_epoch: snapshot.release_epoch,
1992 capacity_epoch: snapshot.capacity_epoch,
1993 available: snapshot,
1994 blockers: evaluation.blockers,
1995 action: match demand.pressure_action {
1996 AdmissionPressureAction::WaitForRelease => DeferredAction::WaitForRelease,
1997 AdmissionPressureAction::PreemptAndRecompute => DeferredAction::PreemptAndRecompute,
1998 },
1999 wait_condition,
2000 }))
2001 }
2002
2003 pub(crate) fn try_claim_for_sequence(
2004 &self,
2005 sequence: &LogicalAdmissionLease,
2006 demand: &AdmissionDemand,
2007 ) -> Result<CapacityClaimDecision, VNextError> {
2008 match self.try_claim_for_sequences(&[sequence], demand)? {
2009 BatchCapacityClaimDecision::Claimed(batch) => {
2010 Ok(CapacityClaimDecision::Claimed(LogicalCapacityLease {
2011 batch,
2012 }))
2013 }
2014 BatchCapacityClaimDecision::Deferred(deferred) => {
2015 Ok(CapacityClaimDecision::Deferred(deferred))
2016 }
2017 BatchCapacityClaimDecision::PermanentRejected(rejected) => {
2018 Ok(CapacityClaimDecision::PermanentRejected(rejected))
2019 }
2020 }
2021 }
2022
2023 pub(crate) fn try_claim_for_sequences(
2028 &self,
2029 sequences: &[&LogicalAdmissionLease],
2030 demand: &AdmissionDemand,
2031 ) -> Result<BatchCapacityClaimDecision, VNextError> {
2032 if sequences.is_empty() || demand.immediate_claim.is_empty() {
2033 return Err(invalid_admission(
2034 "batch child capacity claim requires live parents and non-empty demand",
2035 ));
2036 }
2037 let mut parents = Vec::with_capacity(sequences.len());
2038 for sequence in sequences {
2039 if !self.owns(sequence) {
2040 return Err(admission_fault(
2041 DynamicAdmissionFaultKind::ForeignCoordinator,
2042 "batch parent sequence belongs to another coordinator",
2043 ));
2044 }
2045 if sequence.released {
2046 return Err(invalid_admission(
2047 "batch child capacity claim requires every parent to be live",
2048 ));
2049 }
2050 parents.push(SequenceCapacityParent {
2051 request: sequence.request,
2052 sequence: sequence.sequence,
2053 });
2054 }
2055 u32::try_from(parents.len())
2056 .map_err(|_| invalid_admission("batch parent count exceeds the protocol range"))?;
2057 parents.sort_by_key(|parent| (parent.sequence, parent.request));
2058 if parents.windows(2).any(|pair| pair[0] == pair[1]) {
2059 return Err(invalid_admission(
2060 "batch child capacity claim contains a duplicate parent sequence",
2061 ));
2062 }
2063
2064 let mut state = self.inner.lock_mutation()?;
2065 if state.poisoned {
2066 return Err(admission_fault(
2067 DynamicAdmissionFaultKind::Poisoned,
2068 "coordinator is fail-closed",
2069 ));
2070 }
2071 let mut next_parent_child_claims = Vec::with_capacity(parents.len());
2072 for parent in &parents {
2073 let record = state
2074 .live_sequences
2075 .get(parent.sequence.sparse_id as usize)
2076 .and_then(|record| *record)
2077 .filter(|record| {
2078 record.generation == parent.sequence.generation
2079 && record.request == parent.request
2080 })
2081 .ok_or_else(|| {
2082 invalid_admission("batch parent sequence authority is stale or not live")
2083 })?;
2084 let next = record.active_child_claims.checked_add(1).ok_or_else(|| {
2085 admission_fault(
2086 DynamicAdmissionFaultKind::AuthorityExhausted,
2087 "batch parent child capacity claim count is exhausted",
2088 )
2089 })?;
2090 next_parent_child_claims.push((parent.sequence.sparse_id, next));
2091 }
2092
2093 let evaluation = evaluate_demand(&state, demand)?;
2094 if !evaluation.permanent.is_empty() {
2095 return Ok(BatchCapacityClaimDecision::PermanentRejected(
2096 AdmissionRejected {
2097 immediate_requested: demand.immediate_claim.clone(),
2098 fit_requested: demand.fit_requirement.clone(),
2099 maximum: state.snapshot(self.id()),
2100 blockers: evaluation.permanent,
2101 },
2102 ));
2103 }
2104 if !evaluation.blockers.is_empty() {
2105 let action = deferred_action(demand, evaluation.growth_required);
2106 let wait_condition =
2107 state.wait_condition_for_blockers(self.id(), &evaluation.blockers)?;
2108 let snapshot = state.snapshot(self.id());
2109 return Ok(BatchCapacityClaimDecision::Deferred(AdmissionDeferred {
2110 immediate_requested: demand.immediate_claim.clone(),
2111 fit_requested: demand.fit_requirement.clone(),
2112 release_epoch: snapshot.release_epoch,
2113 capacity_epoch: snapshot.capacity_epoch,
2114 available: snapshot,
2115 blockers: evaluation.blockers,
2116 action,
2117 wait_condition,
2118 }));
2119 }
2120
2121 let committed_claims = demand.immediate_claim.clone();
2122 let mut next_usage = Vec::with_capacity(demand.immediate_claim.entries().len());
2123 for entry in demand.immediate_claim.entries() {
2124 let domain = state
2125 .domains
2126 .get(&entry.domain)
2127 .expect("known batch demand domain was preflight validated");
2128 let used = domain.used.checked_add(entry.units.get()).ok_or_else(|| {
2129 admission_fault(
2130 DynamicAdmissionFaultKind::ArithmeticOverflow,
2131 "batch child capacity usage overflows u64",
2132 )
2133 })?;
2134 next_usage.push((entry.domain, used));
2135 }
2136 let next_child_claims = state.active_child_claims.checked_add(1).ok_or_else(|| {
2137 admission_fault(
2138 DynamicAdmissionFaultKind::AuthorityExhausted,
2139 "active batch child capacity claim count is exhausted",
2140 )
2141 })?;
2142 state
2143 .release_epoch
2144 .checked_add(u64::from(state.active_requests))
2145 .and_then(|epoch| epoch.checked_add(u64::from(state.active_sequences)))
2146 .and_then(|epoch| epoch.checked_add(next_child_claims))
2147 .ok_or_else(|| {
2148 admission_fault(
2149 DynamicAdmissionFaultKind::EpochExhausted,
2150 "release epoch cannot represent every outstanding lease release",
2151 )
2152 })?;
2153
2154 for (domain, used) in next_usage {
2155 state
2156 .domains
2157 .get_mut(&domain)
2158 .expect("validated batch capacity domain remains registered")
2159 .used = used;
2160 }
2161 state.active_child_claims = next_child_claims;
2162 for (sparse_id, next) in next_parent_child_claims {
2163 state.live_sequences[sparse_id as usize]
2164 .as_mut()
2165 .expect("validated batch parent sequence remains live")
2166 .active_child_claims = next;
2167 }
2168 Ok(BatchCapacityClaimDecision::Claimed(
2169 LogicalBatchCapacityLease {
2170 inner: Arc::clone(&self.inner),
2171 parents,
2172 claims: committed_claims,
2173 released: false,
2174 },
2175 ))
2176 }
2177
2178 pub(crate) fn owns_capacity_claim(&self, lease: &LogicalCapacityLease) -> bool {
2179 self.owns_batch_capacity_claim(&lease.batch)
2180 }
2181
2182 pub(crate) fn owns_batch_capacity_claim(&self, lease: &LogicalBatchCapacityLease) -> bool {
2183 self.id() == lease.coordinator_id() && Arc::ptr_eq(&self.inner, &lease.inner)
2184 }
2185
2186 pub fn snapshot(&self) -> Result<CapacitySnapshot, VNextError> {
2187 match self.inner.state.lock() {
2188 Ok(state) => Ok(state.snapshot(self.id())),
2189 Err(poisoned) => {
2190 let mut state = poisoned.into_inner();
2191 state.poisoned = true;
2192 let snapshot = state.snapshot(self.id());
2193 self.inner.epoch_tx.send_replace(state.epochs(self.id()));
2194 Ok(snapshot)
2195 }
2196 }
2197 }
2198
2199 pub fn epochs(&self) -> Result<CapacityEpochs, VNextError> {
2200 let state = self.inner.lock_state()?;
2201 if state.poisoned {
2202 return Err(admission_fault(
2203 DynamicAdmissionFaultKind::Poisoned,
2204 "coordinator is fail-closed",
2205 ));
2206 }
2207 Ok(state.epochs(self.id()))
2208 }
2209
2210 pub(crate) fn subscribe_epochs(&self) -> watch::Receiver<CapacityEpochs> {
2211 self.inner.epoch_tx.subscribe()
2212 }
2213
2214 pub fn write_availability_epochs(
2217 &self,
2218 out: &mut Vec<CapacityAvailabilityEpoch>,
2219 ) -> Result<CapacityEpochs, VNextError> {
2220 let state = self.inner.lock_state()?;
2221 if state.poisoned {
2222 return Err(admission_fault(
2223 DynamicAdmissionFaultKind::Poisoned,
2224 "coordinator is fail-closed",
2225 ));
2226 }
2227 state.write_availability_epochs(out);
2228 Ok(state.epochs(self.id()))
2229 }
2230
2231 pub(crate) fn wait_snapshot_for_domains(
2232 &self,
2233 domains: impl IntoIterator<Item = CapacityDomainId>,
2234 ) -> Result<CapacityWaitSnapshot, VNextError> {
2235 let state = self.inner.lock_state()?;
2236 if state.poisoned {
2237 return Err(admission_fault(
2238 DynamicAdmissionFaultKind::Poisoned,
2239 "coordinator is fail-closed",
2240 ));
2241 }
2242 Ok(CapacityWaitSnapshot::new(
2243 state.epochs(self.id()),
2244 state.wait_condition_for_domains(self.id(), domains)?,
2245 ))
2246 }
2247
2248 pub(crate) fn refresh_wait_snapshot(
2249 &self,
2250 observed: &CapacityWaitCondition,
2251 ) -> Result<CapacityWaitSnapshot, VNextError> {
2252 let state = self.inner.lock_state()?;
2253 if state.poisoned {
2254 return Err(admission_fault(
2255 DynamicAdmissionFaultKind::Poisoned,
2256 "coordinator is fail-closed",
2257 ));
2258 }
2259 Ok(CapacityWaitSnapshot::new(
2260 state.epochs(self.id()),
2261 state.refresh_wait_condition(self.id(), observed)?,
2262 ))
2263 }
2264
2265 pub(crate) fn set_domain_total(
2266 &self,
2267 domain: CapacityDomainId,
2268 new_total: CapacityUnits,
2269 ) -> Result<CapacityEpochs, VNextError> {
2270 self.set_domain_totals(&[(domain, new_total)])
2271 }
2272
2273 pub(crate) fn set_domain_totals(
2274 &self,
2275 updates: &[(CapacityDomainId, CapacityUnits)],
2276 ) -> Result<CapacityEpochs, VNextError> {
2277 let mut state = self.inner.lock_mutation()?;
2278 if state.poisoned {
2279 return Err(admission_fault(
2280 DynamicAdmissionFaultKind::Poisoned,
2281 "coordinator is fail-closed",
2282 ));
2283 }
2284 let mut seen = BTreeSet::new();
2285 let mut changed = false;
2286 for (domain, new_total) in updates {
2287 if !seen.insert(*domain) {
2288 return Err(invalid_admission(
2289 "capacity update contains a duplicate domain",
2290 ));
2291 }
2292 let domain_state = state.domains.get(domain).ok_or_else(|| {
2293 admission_fault(
2294 DynamicAdmissionFaultKind::UnknownDomain,
2295 "capacity update references unknown domain",
2296 )
2297 })?;
2298 if new_total.get() < domain_state.used
2299 || new_total.get() > domain_state.spec.maximum_total_units.get()
2300 {
2301 return Err(invalid_admission(
2302 "capacity update is below live use or above maximum total",
2303 ));
2304 }
2305 if new_total.get() > domain_state.spec.total_units.get()
2306 && domain_state.availability_epoch == u64::MAX
2307 {
2308 return Err(admission_fault(
2309 DynamicAdmissionFaultKind::EpochExhausted,
2310 "domain availability epoch is exhausted",
2311 ));
2312 }
2313 changed |= *new_total != domain_state.spec.total_units;
2314 }
2315 if !changed {
2316 return Ok(state.epochs(self.id()));
2317 }
2318 let next_capacity_epoch = state.capacity_epoch.checked_add(1).ok_or_else(|| {
2319 admission_fault(
2320 DynamicAdmissionFaultKind::EpochExhausted,
2321 "capacity epoch is exhausted",
2322 )
2323 })?;
2324 for (domain, new_total) in updates {
2325 let domain_state = state
2326 .domains
2327 .get_mut(domain)
2328 .expect("validated capacity domain remains registered");
2329 if new_total.get() > domain_state.spec.total_units.get() {
2330 domain_state.availability_epoch += 1;
2331 }
2332 domain_state.spec.total_units = *new_total;
2333 }
2334 state.capacity_epoch = next_capacity_epoch;
2335 let epochs = state.epochs(self.id());
2336 self.inner.epoch_tx.send_replace(epochs);
2337 drop(state);
2338 Ok(epochs)
2339 }
2340
2341 pub(crate) fn notify_domain_availability_changed(
2345 &self,
2346 domain: CapacityDomainId,
2347 ) -> Result<CapacityEpochs, VNextError> {
2348 let mut state = self.inner.lock_mutation()?;
2349 if state.poisoned {
2350 return Err(admission_fault(
2351 DynamicAdmissionFaultKind::Poisoned,
2352 "coordinator is fail-closed",
2353 ));
2354 }
2355 if !state.domains.contains_key(&domain) {
2356 return Err(admission_fault(
2357 DynamicAdmissionFaultKind::UnknownDomain,
2358 "availability update references unknown domain",
2359 ));
2360 }
2361 let next_capacity_epoch = state.capacity_epoch.checked_add(1).ok_or_else(|| {
2362 admission_fault(
2363 DynamicAdmissionFaultKind::EpochExhausted,
2364 "capacity epoch is exhausted",
2365 )
2366 })?;
2367 let domain_state = state
2368 .domains
2369 .get_mut(&domain)
2370 .expect("validated availability domain remains registered");
2371 domain_state.availability_epoch = domain_state
2372 .availability_epoch
2373 .checked_add(1)
2374 .ok_or_else(|| {
2375 admission_fault(
2376 DynamicAdmissionFaultKind::EpochExhausted,
2377 "domain availability epoch is exhausted",
2378 )
2379 })?;
2380 state.capacity_epoch = next_capacity_epoch;
2381 let epochs = state.epochs(self.id());
2382 self.inner.epoch_tx.send_replace(epochs);
2383 drop(state);
2384 Ok(epochs)
2385 }
2386
2387 pub fn register_waiter(
2388 &self,
2389 observed: CapacityWaitCondition,
2390 ) -> Result<CapacityWaitRegistration, VNextError> {
2391 if observed.coordinator_id != self.id() {
2392 return Err(admission_fault(
2393 DynamicAdmissionFaultKind::ForeignCoordinator,
2394 "wait observation belongs to a different coordinator",
2395 ));
2396 }
2397 let receiver = self.inner.epoch_tx.subscribe();
2398 let state = self.inner.lock_state()?;
2399 if state.poisoned {
2400 return Err(admission_fault(
2401 DynamicAdmissionFaultKind::Poisoned,
2402 "coordinator is fail-closed",
2403 ));
2404 }
2405 let registered = state.refresh_wait_condition(self.id(), &observed)?;
2406 Ok(CapacityWaitRegistration {
2407 inner: Arc::clone(&self.inner),
2408 observed,
2409 registered,
2410 receiver,
2411 })
2412 }
2413}
2414
2415fn shortfall(
2416 requested: CapacityEntry,
2417 state: &DomainState,
2418 available: u64,
2419 kind: CapacityShortfallKind,
2420) -> CapacityShortfall {
2421 CapacityShortfall {
2422 domain: Some(requested.domain),
2423 kind,
2424 requested: requested.units,
2425 available: CapacityUnits::new(available),
2426 current_total: state.spec.total_units,
2427 maximum_total: state.spec.maximum_total_units,
2428 }
2429}
2430
2431struct DemandEvaluation {
2432 permanent: Vec<CapacityShortfall>,
2433 blockers: Vec<CapacityShortfall>,
2434 growth_required: bool,
2435}
2436
2437fn evaluate_demand(
2438 state: &CoordinatorState,
2439 demand: &AdmissionDemand,
2440) -> Result<DemandEvaluation, VNextError> {
2441 let mut permanent = Vec::new();
2442 for requested in demand
2443 .immediate_claim
2444 .entries()
2445 .iter()
2446 .chain(demand.fit_requirement.entries())
2447 {
2448 let Some(domain) = state.domains.get(&requested.domain) else {
2449 return Err(admission_fault(
2450 DynamicAdmissionFaultKind::UnknownDomain,
2451 format!(
2452 "demand references unknown capacity domain {}",
2453 requested.domain.get()
2454 ),
2455 ));
2456 };
2457 if requested.units.get() > domain.spec.maximum_total_units.get() {
2458 permanent.push(CapacityShortfall {
2459 domain: Some(requested.domain),
2460 kind: CapacityShortfallKind::PermanentDomainMaximum,
2461 requested: requested.units,
2462 available: CapacityUnits::new(
2463 domain.spec.total_units.get().saturating_sub(domain.used),
2464 ),
2465 current_total: domain.spec.total_units,
2466 maximum_total: domain.spec.maximum_total_units,
2467 });
2468 }
2469 }
2470 permanent.sort_by_key(|shortfall| (shortfall.domain, shortfall.kind as u8));
2471 permanent.dedup_by(|left, right| {
2472 left.domain == right.domain && left.kind == right.kind && left.requested == right.requested
2473 });
2474
2475 let mut blockers = Vec::new();
2476 let mut growth_required = false;
2477 if permanent.is_empty() {
2478 for requested in demand.immediate_claim.entries() {
2479 let domain = state
2480 .domains
2481 .get(&requested.domain)
2482 .expect("known demand domain was preflight validated");
2483 let available = domain.spec.total_units.get().saturating_sub(domain.used);
2484 if requested.units.get() > domain.spec.total_units.get() {
2485 growth_required = true;
2486 blockers.push(shortfall(
2487 *requested,
2488 domain,
2489 available,
2490 CapacityShortfallKind::BackingGrowthRequired,
2491 ));
2492 } else if requested.units.get() > available {
2493 blockers.push(shortfall(
2494 *requested,
2495 domain,
2496 available,
2497 CapacityShortfallKind::ImmediateAvailability,
2498 ));
2499 }
2500 }
2501 if demand.fit_policy == AdmissionFitPolicy::FullInputMustFit {
2502 for requested in demand.fit_requirement.entries() {
2503 let domain = state
2504 .domains
2505 .get(&requested.domain)
2506 .expect("known fit domain was preflight validated");
2507 let available = domain.spec.total_units.get().saturating_sub(domain.used);
2508 if requested.units.get() > domain.spec.total_units.get() {
2509 growth_required = true;
2510 blockers.push(shortfall(
2511 *requested,
2512 domain,
2513 available,
2514 CapacityShortfallKind::BackingGrowthRequired,
2515 ));
2516 } else if requested.units.get() > available {
2517 blockers.push(shortfall(
2518 *requested,
2519 domain,
2520 available,
2521 CapacityShortfallKind::FitAvailability,
2522 ));
2523 }
2524 }
2525 }
2526 }
2527 Ok(DemandEvaluation {
2528 permanent,
2529 blockers,
2530 growth_required,
2531 })
2532}
2533
2534fn deferred_action(demand: &AdmissionDemand, growth_required: bool) -> DeferredAction {
2535 if growth_required {
2536 DeferredAction::AwaitBackingGrowth
2537 } else {
2538 match demand.pressure_action {
2539 AdmissionPressureAction::WaitForRelease => DeferredAction::WaitForRelease,
2540 AdmissionPressureAction::PreemptAndRecompute => DeferredAction::PreemptAndRecompute,
2541 }
2542 }
2543}
2544
2545#[derive(Debug)]
2546#[must_use = "dropping the request lease releases request-scoped capacity"]
2547pub struct LogicalRequestLease {
2548 inner: Arc<CoordinatorInner>,
2549 request: RequestAuthorityId,
2550 claims: CapacityVector,
2551 released: bool,
2552}
2553
2554impl LogicalRequestLease {
2555 pub fn coordinator_id(&self) -> LogicalAdmissionCoordinatorId {
2556 self.inner.id
2557 }
2558
2559 pub const fn request(&self) -> RequestAuthorityId {
2560 self.request
2561 }
2562
2563 pub fn claims(&self) -> &CapacityVector {
2564 &self.claims
2565 }
2566
2567 fn release_inner(&mut self) -> bool {
2568 if self.released {
2569 return true;
2570 }
2571 let state = match self.inner.state.lock() {
2572 Ok(state) => state,
2573 Err(poisoned) => {
2574 let mut state = poisoned.into_inner();
2575 state.poisoned = true;
2576 let epochs = state.epochs(self.coordinator_id());
2577 self.inner.epoch_tx.send_replace(epochs);
2578 return false;
2579 }
2580 };
2581 let mut state = CoordinatorMutationGuard {
2582 inner: self.inner.as_ref(),
2583 state,
2584 panicking_on_entry: std::thread::panicking(),
2585 };
2586 let request_is_releasable = state
2587 .live_requests
2588 .get(self.request.sparse_id as usize)
2589 .and_then(|record| *record)
2590 .is_some_and(|record| {
2591 record.generation == self.request.generation && record.active_sequences == 0
2592 });
2593 if state.poisoned
2594 || !request_is_releasable
2595 || state.active_requests == 0
2596 || self.claims.entries().iter().any(|claim| {
2597 state
2598 .domains
2599 .get(&claim.domain)
2600 .is_none_or(|domain| domain.used < claim.units.get())
2601 })
2602 {
2603 state.poisoned = true;
2604 let epochs = state.epochs(self.coordinator_id());
2605 self.inner.epoch_tx.send_replace(epochs);
2606 return false;
2607 }
2608 let Some(next_release_epoch) = state.release_epoch.checked_add(1) else {
2609 state.poisoned = true;
2610 let epochs = state.epochs(self.coordinator_id());
2611 self.inner.epoch_tx.send_replace(epochs);
2612 return false;
2613 };
2614 if self.claims.entries().iter().any(|claim| {
2615 state
2616 .domains
2617 .get(&claim.domain)
2618 .is_some_and(|domain| domain.availability_epoch == u64::MAX)
2619 }) {
2620 state.poisoned = true;
2621 let epochs = state.epochs(self.coordinator_id());
2622 self.inner.epoch_tx.send_replace(epochs);
2623 return false;
2624 }
2625 for claim in self.claims.entries() {
2626 let domain = state
2627 .domains
2628 .get_mut(&claim.domain)
2629 .expect("validated request capacity domain remains registered");
2630 domain.used -= claim.units.get();
2631 domain.availability_epoch += 1;
2632 }
2633 state.active_requests -= 1;
2634 state.live_requests[self.request.sparse_id as usize] = None;
2635 state.reusable_request_ids.push(self.request.sparse_id);
2636 state.release_epoch = next_release_epoch;
2637 let epochs = state.epochs(self.coordinator_id());
2638 self.inner.epoch_tx.send_replace(epochs);
2639 drop(state);
2640 self.released = true;
2641 true
2642 }
2643}
2644
2645impl Drop for LogicalRequestLease {
2646 fn drop(&mut self) {
2647 let _ = self.release_inner();
2648 }
2649}
2650
2651#[derive(Debug)]
2652#[must_use = "dropping the logical admission lease releases its capacity claim"]
2653pub struct LogicalAdmissionLease {
2654 inner: Arc<CoordinatorInner>,
2655 request: RequestAuthorityId,
2656 sequence: SequenceAuthorityId,
2657 claims: CapacityVector,
2658 released: bool,
2659}
2660
2661impl LogicalAdmissionLease {
2662 pub fn coordinator_id(&self) -> LogicalAdmissionCoordinatorId {
2663 self.inner.id
2664 }
2665
2666 pub const fn sequence(&self) -> SequenceAuthorityId {
2667 self.sequence
2668 }
2669
2670 pub const fn request(&self) -> RequestAuthorityId {
2671 self.request
2672 }
2673
2674 pub fn claims(&self) -> &CapacityVector {
2675 &self.claims
2676 }
2677
2678 fn release_inner(&mut self) -> bool {
2679 if self.released {
2680 return true;
2681 }
2682 let state = match self.inner.state.lock() {
2683 Ok(state) => state,
2684 Err(poisoned) => {
2685 let mut state = poisoned.into_inner();
2686 state.poisoned = true;
2687 let epochs = state.epochs(self.coordinator_id());
2688 self.inner.epoch_tx.send_replace(epochs);
2689 return false;
2690 }
2691 };
2692 let mut state = CoordinatorMutationGuard {
2693 inner: self.inner.as_ref(),
2694 state,
2695 panicking_on_entry: std::thread::panicking(),
2696 };
2697 let sequence_record = state
2698 .live_sequences
2699 .get(self.sequence.sparse_id as usize)
2700 .and_then(|record| *record);
2701 let request_record = state
2702 .live_requests
2703 .get(self.request.sparse_id as usize)
2704 .and_then(|record| *record);
2705 if state.poisoned
2706 || sequence_record.is_none_or(|record| {
2707 record.generation != self.sequence.generation
2708 || record.request != self.request
2709 || record.active_child_claims != 0
2710 })
2711 || request_record.is_none_or(|record| {
2712 record.generation != self.request.generation || record.active_sequences == 0
2713 })
2714 || state.active_sequences == 0
2715 || self.claims.entries().iter().any(|claim| {
2716 state
2717 .domains
2718 .get(&claim.domain)
2719 .is_none_or(|domain| domain.used < claim.units.get())
2720 })
2721 {
2722 state.poisoned = true;
2723 let epochs = state.epochs(self.coordinator_id());
2724 self.inner.epoch_tx.send_replace(epochs);
2725 return false;
2726 }
2727 let Some(next_release_epoch) = state.release_epoch.checked_add(1) else {
2728 state.poisoned = true;
2729 let epochs = state.epochs(self.coordinator_id());
2730 self.inner.epoch_tx.send_replace(epochs);
2731 return false;
2732 };
2733 if state.active_sequence_availability_epoch == u64::MAX
2734 || self.claims.entries().iter().any(|claim| {
2735 state
2736 .domains
2737 .get(&claim.domain)
2738 .is_some_and(|domain| domain.availability_epoch == u64::MAX)
2739 })
2740 {
2741 state.poisoned = true;
2742 let epochs = state.epochs(self.coordinator_id());
2743 self.inner.epoch_tx.send_replace(epochs);
2744 return false;
2745 }
2746 for claim in self.claims.entries() {
2747 let domain = state
2748 .domains
2749 .get_mut(&claim.domain)
2750 .expect("validated logical claim domain remains registered");
2751 domain.used -= claim.units.get();
2752 domain.availability_epoch += 1;
2753 }
2754 state.active_sequences -= 1;
2755 state.active_sequence_availability_epoch += 1;
2756 state.live_requests[self.request.sparse_id as usize]
2757 .as_mut()
2758 .expect("validated parent request remains live")
2759 .active_sequences -= 1;
2760 state.live_sequences[self.sequence.sparse_id as usize] = None;
2761 state.reusable_sequence_ids.push(self.sequence.sparse_id);
2762 state.release_epoch = next_release_epoch;
2763 let epochs = state.epochs(self.coordinator_id());
2764 self.inner.epoch_tx.send_replace(epochs);
2765 drop(state);
2766 self.released = true;
2767 true
2768 }
2769}
2770
2771impl Drop for LogicalAdmissionLease {
2772 fn drop(&mut self) {
2773 let _ = self.release_inner();
2774 }
2775}
2776
2777#[derive(Debug)]
2778#[must_use = "dropping a child capacity lease releases its exact domain claims"]
2779pub struct LogicalCapacityLease {
2780 batch: LogicalBatchCapacityLease,
2781}
2782
2783impl LogicalCapacityLease {
2784 pub fn coordinator_id(&self) -> LogicalAdmissionCoordinatorId {
2785 self.batch.coordinator_id()
2786 }
2787
2788 pub fn sequence(&self) -> SequenceAuthorityId {
2789 self.batch.parents[0].sequence
2790 }
2791
2792 pub fn request(&self) -> RequestAuthorityId {
2793 self.batch.parents[0].request
2794 }
2795
2796 pub fn claims(&self) -> &CapacityVector {
2797 self.batch.claims()
2798 }
2799}
2800
2801#[derive(Debug)]
2802#[must_use = "dropping a batch child lease releases its exact shared claim"]
2803pub struct LogicalBatchCapacityLease {
2804 inner: Arc<CoordinatorInner>,
2805 parents: Vec<SequenceCapacityParent>,
2806 claims: CapacityVector,
2807 released: bool,
2808}
2809
2810impl LogicalBatchCapacityLease {
2811 pub fn coordinator_id(&self) -> LogicalAdmissionCoordinatorId {
2812 self.inner.id
2813 }
2814
2815 pub fn parents(&self) -> &[SequenceCapacityParent] {
2816 &self.parents
2817 }
2818
2819 pub fn claims(&self) -> &CapacityVector {
2820 &self.claims
2821 }
2822
2823 fn release_inner(&mut self) -> bool {
2824 if self.released {
2825 return true;
2826 }
2827 let state = match self.inner.state.lock() {
2828 Ok(state) => state,
2829 Err(poisoned) => {
2830 let mut state = poisoned.into_inner();
2831 state.poisoned = true;
2832 let epochs = state.epochs(self.coordinator_id());
2833 self.inner.epoch_tx.send_replace(epochs);
2834 return false;
2835 }
2836 };
2837 let mut state = CoordinatorMutationGuard {
2838 inner: self.inner.as_ref(),
2839 state,
2840 panicking_on_entry: std::thread::panicking(),
2841 };
2842 let parents_are_live = !self.parents.is_empty()
2843 && self.parents.iter().all(|parent| {
2844 state
2845 .live_sequences
2846 .get(parent.sequence.sparse_id as usize)
2847 .and_then(|record| *record)
2848 .is_some_and(|record| {
2849 record.generation == parent.sequence.generation
2850 && record.request == parent.request
2851 && record.active_child_claims > 0
2852 })
2853 });
2854 if state.poisoned
2855 || !parents_are_live
2856 || state.active_child_claims == 0
2857 || self.claims.entries().iter().any(|claim| {
2858 state
2859 .domains
2860 .get(&claim.domain)
2861 .is_none_or(|domain| domain.used < claim.units.get())
2862 })
2863 {
2864 state.poisoned = true;
2865 let epochs = state.epochs(self.coordinator_id());
2866 self.inner.epoch_tx.send_replace(epochs);
2867 return false;
2868 }
2869 let Some(next_release_epoch) = state.release_epoch.checked_add(1) else {
2870 state.poisoned = true;
2871 let epochs = state.epochs(self.coordinator_id());
2872 self.inner.epoch_tx.send_replace(epochs);
2873 return false;
2874 };
2875 if self.claims.entries().iter().any(|claim| {
2876 state
2877 .domains
2878 .get(&claim.domain)
2879 .is_some_and(|domain| domain.availability_epoch == u64::MAX)
2880 }) {
2881 state.poisoned = true;
2882 let epochs = state.epochs(self.coordinator_id());
2883 self.inner.epoch_tx.send_replace(epochs);
2884 return false;
2885 }
2886 for claim in self.claims.entries() {
2887 let domain = state
2888 .domains
2889 .get_mut(&claim.domain)
2890 .expect("validated child capacity domain remains registered");
2891 domain.used -= claim.units.get();
2892 domain.availability_epoch += 1;
2893 }
2894 state.active_child_claims -= 1;
2895 for parent in &self.parents {
2896 state.live_sequences[parent.sequence.sparse_id as usize]
2897 .as_mut()
2898 .expect("validated batch parent sequence remains live")
2899 .active_child_claims -= 1;
2900 }
2901 state.release_epoch = next_release_epoch;
2902 let epochs = state.epochs(self.coordinator_id());
2903 self.inner.epoch_tx.send_replace(epochs);
2904 drop(state);
2905 self.released = true;
2906 true
2907 }
2908}
2909
2910impl Drop for LogicalBatchCapacityLease {
2911 fn drop(&mut self) {
2912 let _ = self.release_inner();
2913 }
2914}
2915
2916#[derive(Debug)]
2917pub struct CapacityWaitRegistration {
2918 inner: Arc<CoordinatorInner>,
2919 observed: CapacityWaitCondition,
2920 registered: CapacityWaitCondition,
2921 receiver: watch::Receiver<CapacityEpochs>,
2922}
2923
2924#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2925pub struct CapacityWaitRecheck {
2926 current: CapacityEpochs,
2927 changed_since_observation: bool,
2928 changed_since_registration: bool,
2929}
2930
2931impl CapacityWaitRegistration {
2932 pub fn recheck(&self) -> Result<CapacityWaitRecheck, VNextError> {
2933 let state = self.inner.lock_state()?;
2934 if state.poisoned {
2935 return Err(admission_fault(
2936 DynamicAdmissionFaultKind::Poisoned,
2937 "coordinator is fail-closed",
2938 ));
2939 }
2940 let current = state.epochs(self.inner.id);
2941 Ok(CapacityWaitRecheck {
2942 current,
2943 changed_since_observation: state
2944 .wait_condition_changed(self.inner.id, &self.observed)?,
2945 changed_since_registration: state
2946 .wait_condition_changed(self.inner.id, &self.registered)?,
2947 })
2948 }
2949
2950 pub async fn wait_for_change(mut self) -> Result<CapacityEpochs, VNextError> {
2951 loop {
2952 let recheck = self.recheck()?;
2953 if recheck.should_retry() {
2954 return Ok(recheck.current());
2955 }
2956 self.receiver.changed().await.map_err(|_| {
2957 admission_fault(
2958 DynamicAdmissionFaultKind::Poisoned,
2959 "capacity epoch listener closed",
2960 )
2961 })?;
2962 self.receiver.borrow_and_update();
2963 }
2964 }
2965}
2966
2967impl CapacityWaitRecheck {
2968 pub(crate) const fn new(
2969 current: CapacityEpochs,
2970 changed_since_observation: bool,
2971 changed_since_registration: bool,
2972 ) -> Self {
2973 Self {
2974 current,
2975 changed_since_observation,
2976 changed_since_registration,
2977 }
2978 }
2979
2980 pub const fn current(self) -> CapacityEpochs {
2981 self.current
2982 }
2983
2984 pub const fn changed_since_observation(self) -> bool {
2985 self.changed_since_observation
2986 }
2987
2988 pub const fn changed_since_registration(self) -> bool {
2989 self.changed_since_registration
2990 }
2991
2992 pub const fn should_retry(self) -> bool {
2993 self.changed_since_observation || self.changed_since_registration
2994 }
2995}
2996
2997#[cfg(test)]
2998mod tests {
2999 use super::*;
3000 use std::sync::{Arc, Condvar, Mutex as StdMutex};
3001 use std::thread;
3002
3003 const TEST_NATIVE_WORKER_LIMIT: usize = 8;
3004
3005 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
3006 enum TestGatePhase {
3007 Holding,
3008 Released,
3009 Cancelled,
3010 }
3011
3012 #[derive(Debug)]
3013 struct TestGateState {
3014 ready: usize,
3015 phase: TestGatePhase,
3016 }
3017
3018 #[derive(Debug)]
3019 struct CancellableTestGate {
3020 state: StdMutex<TestGateState>,
3021 changed: Condvar,
3022 }
3023
3024 impl CancellableTestGate {
3025 fn new(worker_count: usize) -> Arc<Self> {
3026 assert!(worker_count > 0);
3027 assert!(worker_count <= TEST_NATIVE_WORKER_LIMIT);
3028 Arc::new(Self {
3029 state: StdMutex::new(TestGateState {
3030 ready: 0,
3031 phase: TestGatePhase::Holding,
3032 }),
3033 changed: Condvar::new(),
3034 })
3035 }
3036
3037 fn arrive_and_wait(&self) -> bool {
3038 let mut state = self.state.lock().unwrap();
3039 if state.phase != TestGatePhase::Holding {
3040 return state.phase == TestGatePhase::Released;
3041 }
3042 state.ready += 1;
3043 self.changed.notify_all();
3044 while state.phase == TestGatePhase::Holding {
3045 state = self.changed.wait(state).unwrap();
3046 }
3047 state.phase == TestGatePhase::Released
3048 }
3049
3050 fn wait_until_ready(&self, worker_count: usize) -> bool {
3051 assert!(worker_count <= TEST_NATIVE_WORKER_LIMIT);
3052 let mut state = self.state.lock().unwrap();
3053 while state.ready < worker_count && state.phase == TestGatePhase::Holding {
3054 state = self.changed.wait(state).unwrap();
3055 }
3056 state.ready == worker_count && state.phase == TestGatePhase::Holding
3057 }
3058
3059 fn release(&self) {
3060 let mut state = self.state.lock().unwrap();
3061 assert_eq!(state.phase, TestGatePhase::Holding);
3062 state.phase = TestGatePhase::Released;
3063 self.changed.notify_all();
3064 }
3065
3066 fn cancel(&self) {
3067 let mut state = self
3068 .state
3069 .lock()
3070 .unwrap_or_else(|poisoned| poisoned.into_inner());
3071 if state.phase == TestGatePhase::Holding {
3072 state.phase = TestGatePhase::Cancelled;
3073 self.changed.notify_all();
3074 }
3075 }
3076 }
3077
3078 struct CancelTestGateOnDrop<'a> {
3079 gate: &'a CancellableTestGate,
3080 armed: bool,
3081 }
3082
3083 impl<'a> CancelTestGateOnDrop<'a> {
3084 fn new(gate: &'a CancellableTestGate) -> Self {
3085 Self { gate, armed: true }
3086 }
3087
3088 fn disarm(&mut self) {
3089 self.armed = false;
3090 }
3091 }
3092
3093 impl Drop for CancelTestGateOnDrop<'_> {
3094 fn drop(&mut self) {
3095 if self.armed {
3096 self.gate.cancel();
3097 }
3098 }
3099 }
3100
3101 fn cancel_gate_on_unwind<T>(gate: &CancellableTestGate, work: impl FnOnce() -> T) -> T {
3102 match std::panic::catch_unwind(std::panic::AssertUnwindSafe(work)) {
3103 Ok(value) => value,
3104 Err(payload) => {
3105 gate.cancel();
3106 std::panic::resume_unwind(payload);
3107 }
3108 }
3109 }
3110
3111 fn domain(value: u32) -> CapacityDomainId {
3112 CapacityDomainId::new(value).unwrap()
3113 }
3114
3115 fn vector(entries: &[(u32, u64)]) -> CapacityVector {
3116 CapacityVector::new(
3117 entries
3118 .iter()
3119 .map(|(domain_id, units)| {
3120 CapacityEntry::new(domain(*domain_id), CapacityUnits::new(*units)).unwrap()
3121 })
3122 .collect(),
3123 )
3124 .unwrap()
3125 }
3126
3127 fn demand(immediate: &[(u32, u64)], fit: &[(u32, u64)]) -> AdmissionDemand {
3128 AdmissionDemand::from_plan(
3129 vector(immediate),
3130 vector(fit),
3131 AdmissionFitPolicy::FullInputMustFit,
3132 AdmissionPressureAction::WaitForRelease,
3133 )
3134 .unwrap()
3135 }
3136
3137 fn coordinator(maximum_active_sequences: u32) -> LogicalAdmissionCoordinator {
3138 LogicalAdmissionCoordinator::new(
3139 vec![
3140 (
3141 domain(1),
3142 CapacityDomainSpec::new(CapacityUnits::new(10), CapacityUnits::new(20))
3143 .unwrap(),
3144 ),
3145 (
3146 domain(2),
3147 CapacityDomainSpec::new(CapacityUnits::new(4), CapacityUnits::new(4)).unwrap(),
3148 ),
3149 ],
3150 maximum_active_sequences,
3151 )
3152 .unwrap()
3153 }
3154
3155 fn wait_for_domains(
3156 coordinator: &LogicalAdmissionCoordinator,
3157 domains: &[u32],
3158 ) -> CapacityWaitCondition {
3159 coordinator
3160 .wait_snapshot_for_domains(domains.iter().copied().map(domain))
3161 .unwrap()
3162 .wait_condition()
3163 .clone()
3164 }
3165
3166 fn admitted(decision: AdmissionDecision) -> LogicalAdmissionLease {
3167 match decision {
3168 AdmissionDecision::Admitted(lease) => lease,
3169 _ => panic!("expected admitted decision"),
3170 }
3171 }
3172
3173 fn empty_demand() -> AdmissionDemand {
3174 AdmissionDemand::from_plan(
3175 CapacityVector::empty(),
3176 CapacityVector::empty(),
3177 AdmissionFitPolicy::ImmediateOnly,
3178 AdmissionPressureAction::WaitForRelease,
3179 )
3180 .unwrap()
3181 }
3182
3183 fn admitted_request(decision: RequestAdmissionDecision) -> LogicalRequestLease {
3184 match decision {
3185 RequestAdmissionDecision::Admitted(lease) => lease,
3186 _ => panic!("expected admitted request"),
3187 }
3188 }
3189
3190 fn request(coordinator: &LogicalAdmissionCoordinator) -> LogicalRequestLease {
3191 admitted_request(coordinator.try_admit_request(&empty_demand()).unwrap())
3192 }
3193
3194 fn admit_sequence(
3195 coordinator: &LogicalAdmissionCoordinator,
3196 request: &LogicalRequestLease,
3197 demand: &AdmissionDemand,
3198 ) -> LogicalAdmissionLease {
3199 admitted(
3200 coordinator
3201 .try_admit_sequence_for_request(request, demand)
3202 .unwrap(),
3203 )
3204 }
3205
3206 fn claimed_child(decision: CapacityClaimDecision) -> LogicalCapacityLease {
3207 match decision {
3208 CapacityClaimDecision::Claimed(lease) => lease,
3209 _ => panic!("expected child capacity claim"),
3210 }
3211 }
3212
3213 fn claimed_batch(decision: BatchCapacityClaimDecision) -> LogicalBatchCapacityLease {
3214 match decision {
3215 BatchCapacityClaimDecision::Claimed(lease) => lease,
3216 _ => panic!("expected batch child capacity claim"),
3217 }
3218 }
3219
3220 #[test]
3221 fn request_capacity_is_shared_once_across_multiple_sequences() {
3222 let coordinator = coordinator(2);
3223 let request = admitted_request(
3224 coordinator
3225 .try_admit_request(&demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]))
3226 .unwrap(),
3227 );
3228 let first = admit_sequence(
3229 &coordinator,
3230 &request,
3231 &demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]),
3232 );
3233 let second = admit_sequence(
3234 &coordinator,
3235 &request,
3236 &demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]),
3237 );
3238 let both = coordinator.snapshot().unwrap();
3239 assert_eq!(both.active_requests(), 1);
3240 assert_eq!(both.active_sequences(), 2);
3241 assert_eq!(both.domains()[0].used().get(), 6);
3242 assert_eq!(both.domains()[1].used().get(), 3);
3243 assert_eq!(first.request(), request.request());
3244 assert_eq!(second.request(), request.request());
3245
3246 drop(first);
3247 let one = coordinator.snapshot().unwrap();
3248 assert_eq!(one.active_requests(), 1);
3249 assert_eq!(one.active_sequences(), 1);
3250 assert_eq!(one.domains()[0].used().get(), 4);
3251 assert_eq!(one.domains()[1].used().get(), 2);
3252
3253 drop(second);
3254 let request_only = coordinator.snapshot().unwrap();
3255 assert_eq!(request_only.active_requests(), 1);
3256 assert_eq!(request_only.active_sequences(), 0);
3257 assert_eq!(request_only.domains()[0].used().get(), 2);
3258 assert_eq!(request_only.domains()[1].used().get(), 1);
3259 drop(request);
3260 let empty = coordinator.snapshot().unwrap();
3261 assert_eq!(empty.active_requests(), 0);
3262 assert!(empty
3263 .domains()
3264 .iter()
3265 .all(|domain| domain.used().get() == 0));
3266 }
3267
3268 #[test]
3269 fn request_defer_and_reject_are_atomic_and_do_not_consume_sequence_slots() {
3270 let coordinator = coordinator(2);
3271 let held = admitted_request(
3272 coordinator
3273 .try_admit_request(&demand(&[(1, 1), (2, 3)], &[(1, 1), (2, 3)]))
3274 .unwrap(),
3275 );
3276 let before = coordinator.snapshot().unwrap();
3277 let deferred = coordinator
3278 .try_admit_request(&demand(&[(1, 1), (2, 2)], &[(1, 1), (2, 2)]))
3279 .unwrap();
3280 assert!(matches!(deferred, RequestAdmissionDecision::Deferred(_)));
3281 let rejected = coordinator
3282 .try_admit_request(&demand(&[(1, 21), (2, 1)], &[(1, 21), (2, 1)]))
3283 .unwrap();
3284 assert!(matches!(
3285 rejected,
3286 RequestAdmissionDecision::PermanentRejected(_)
3287 ));
3288 let after = coordinator.snapshot().unwrap();
3289 assert_eq!(before.domains, after.domains);
3290 assert_eq!(after.active_requests(), 1);
3291 assert_eq!(after.active_sequences(), 0);
3292 drop(held);
3293 }
3294
3295 #[test]
3296 fn initial_sequence_defer_retains_no_partial_request_or_capacity() {
3297 let coordinator = coordinator(2);
3298 let held = match coordinator
3299 .try_admit_initial_sequence(
3300 &demand(&[(1, 1)], &[(1, 1)]),
3301 &demand(&[(2, 3)], &[(2, 3)]),
3302 )
3303 .unwrap()
3304 {
3305 InitialSequenceAdmissionDecision::Admitted(bundle) => bundle,
3306 _ => panic!("first initial bundle must be admitted"),
3307 };
3308 let before = coordinator.snapshot().unwrap();
3309
3310 assert!(matches!(
3311 coordinator
3312 .try_admit_initial_sequence(
3313 &demand(&[(1, 1)], &[(1, 1)]),
3314 &demand(&[(2, 2)], &[(2, 2)]),
3315 )
3316 .unwrap(),
3317 InitialSequenceAdmissionDecision::Deferred
3318 ));
3319 let after = coordinator.snapshot().unwrap();
3320 assert_eq!(after.active_requests(), before.active_requests());
3321 assert_eq!(after.active_sequences(), before.active_sequences());
3322 assert_eq!(after.domains, before.domains);
3323
3324 drop(held);
3325 let empty = coordinator.snapshot().unwrap();
3326 assert_eq!(empty.active_requests(), 0);
3327 assert_eq!(empty.active_sequences(), 0);
3328 assert!(empty
3329 .domains()
3330 .iter()
3331 .all(|domain| domain.used().get() == 0));
3332 }
3333
3334 #[test]
3335 fn initial_sequence_bundle_sums_overlapping_domains_and_releases_child_first() {
3336 let coordinator = coordinator(1);
3337 let bundle = match coordinator
3338 .try_admit_initial_sequence(
3339 &demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]),
3340 &demand(&[(1, 3), (2, 2)], &[(1, 3), (2, 2)]),
3341 )
3342 .unwrap()
3343 {
3344 InitialSequenceAdmissionDecision::Admitted(bundle) => bundle,
3345 _ => panic!("overlapping initial bundle must be admitted atomically"),
3346 };
3347 let admitted = coordinator.snapshot().unwrap();
3348 assert_eq!(admitted.active_requests(), 1);
3349 assert_eq!(admitted.active_sequences(), 1);
3350 assert_eq!(admitted.domains()[0].used().get(), 5);
3351 assert_eq!(admitted.domains()[1].used().get(), 3);
3352
3353 let (request, sequence) = bundle.into_parts();
3354 assert_eq!(request.request(), sequence.request());
3355 drop(sequence);
3356 drop(request);
3357 let released = coordinator.snapshot().unwrap();
3358 assert!(!released.poisoned());
3359 assert_eq!(released.active_requests(), 0);
3360 assert_eq!(released.active_sequences(), 0);
3361 assert!(released
3362 .domains()
3363 .iter()
3364 .all(|domain| domain.used().get() == 0));
3365 }
3366
3367 #[test]
3368 fn request_authority_reuses_sparse_storage_with_new_generation() {
3369 let coordinator = coordinator(u32::MAX);
3370 let mut previous_generation = 0;
3371 for _ in 0..100_000 {
3372 let request = request(&coordinator);
3373 assert_eq!(request.request().sparse_id(), 0);
3374 assert!(request.request().generation() > previous_generation);
3375 previous_generation = request.request().generation();
3376 drop(request);
3377 }
3378 let snapshot = coordinator.snapshot().unwrap();
3379 assert_eq!(snapshot.active_requests(), 0);
3380 assert_eq!(snapshot.live_request_records(), 0);
3381 assert_eq!(snapshot.reusable_request_ids(), 1);
3382 assert_eq!(snapshot.release_epoch(), 100_001);
3383 }
3384
3385 #[test]
3386 fn early_request_release_with_live_sequence_fails_closed_and_retains_claims() {
3387 let coordinator = coordinator(1);
3388 let mut request = admitted_request(
3389 coordinator
3390 .try_admit_request(&demand(&[(1, 2)], &[(1, 2)]))
3391 .unwrap(),
3392 );
3393 let sequence = admit_sequence(&coordinator, &request, &demand(&[(1, 1)], &[(1, 1)]));
3394 assert!(!request.release_inner());
3395 let snapshot = coordinator.snapshot().unwrap();
3396 assert!(snapshot.poisoned());
3397 assert_eq!(snapshot.active_requests(), 1);
3398 assert_eq!(snapshot.active_sequences(), 1);
3399 assert_eq!(snapshot.domains()[0].used().get(), 3);
3400 drop(sequence);
3401 drop(request);
3402 }
3403
3404 #[test]
3405 fn exact_fit_claims_only_immediate_and_release_retries() {
3406 let coordinator = coordinator(2);
3407 let request = request(&coordinator);
3408 let first = admit_sequence(
3409 &coordinator,
3410 &request,
3411 &demand(&[(1, 6), (2, 2)], &[(1, 10), (2, 2)]),
3412 );
3413 let snapshot = coordinator.snapshot().unwrap();
3414 assert_eq!(snapshot.domains()[0].used().get(), 6);
3415 assert_eq!(snapshot.domains()[0].available().get(), 4);
3416
3417 let deferred = coordinator
3418 .try_admit_sequence_for_request(&request, &demand(&[(1, 5), (2, 1)], &[(1, 5), (2, 1)]))
3419 .unwrap();
3420 let observed = match deferred {
3421 AdmissionDecision::Deferred(value) => {
3422 assert_eq!(value.action(), DeferredAction::WaitForRelease);
3423 value.wait_condition().clone()
3424 }
3425 _ => panic!("expected capacity defer"),
3426 };
3427 drop(first);
3428 let registration = coordinator.register_waiter(observed).unwrap();
3429 assert!(registration.recheck().unwrap().should_retry());
3430 let retry = coordinator
3431 .try_admit_sequence_for_request(&request, &demand(&[(1, 5), (2, 1)], &[(1, 5), (2, 1)]))
3432 .unwrap();
3433 assert!(matches!(retry, AdmissionDecision::Admitted(_)));
3434 }
3435
3436 #[test]
3437 fn child_claim_uses_parent_authority_without_consuming_sequence_slot() {
3438 let coordinator = coordinator(1);
3439 let request = request(&coordinator);
3440 let parent = admit_sequence(
3441 &coordinator,
3442 &request,
3443 &demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]),
3444 );
3445 let before = coordinator.snapshot().unwrap();
3446 let child = claimed_child(
3447 coordinator
3448 .try_claim_for_sequence(&parent, &demand(&[(1, 3), (2, 1)], &[(1, 3), (2, 1)]))
3449 .unwrap(),
3450 );
3451 assert_eq!(child.sequence(), parent.sequence());
3452 assert!(coordinator.owns_capacity_claim(&child));
3453 let claimed = coordinator.snapshot().unwrap();
3454 assert_eq!(claimed.active_sequences(), 1);
3455 assert_eq!(claimed.active_child_claims(), 1);
3456 assert_eq!(claimed.domains()[0].used().get(), 5);
3457 assert_eq!(claimed.domains()[1].used().get(), 2);
3458
3459 drop(child);
3460 let released = coordinator.snapshot().unwrap();
3461 assert_eq!(released.active_sequences(), 1);
3462 assert_eq!(released.active_child_claims(), 0);
3463 assert_eq!(released.domains()[0].used().get(), 2);
3464 assert_eq!(released.domains()[1].used().get(), 1);
3465 assert_eq!(released.release_epoch(), before.release_epoch() + 1);
3466 drop(parent);
3467 assert_eq!(coordinator.snapshot().unwrap().active_sequences(), 0);
3468 }
3469
3470 #[test]
3471 fn batch_child_claim_charges_once_and_binds_every_parent() {
3472 let coordinator = coordinator(2);
3473 let first_request = request(&coordinator);
3474 let second_request = request(&coordinator);
3475 let first = admit_sequence(&coordinator, &first_request, &empty_demand());
3476 let second = admit_sequence(&coordinator, &second_request, &empty_demand());
3477 let before = coordinator.snapshot().unwrap();
3478
3479 let batch = claimed_batch(
3480 coordinator
3481 .try_claim_for_sequences(
3482 &[&second, &first],
3483 &demand(&[(1, 3), (2, 1)], &[(1, 3), (2, 1)]),
3484 )
3485 .unwrap(),
3486 );
3487 assert!(coordinator.owns_batch_capacity_claim(&batch));
3488 assert_eq!(batch.parents().len(), 2);
3489 assert!(batch
3490 .parents()
3491 .windows(2)
3492 .all(|pair| pair[0].sequence() < pair[1].sequence()));
3493 let claimed = coordinator.snapshot().unwrap();
3494 assert_eq!(claimed.active_requests(), 2);
3495 assert_eq!(claimed.active_sequences(), 2);
3496 assert_eq!(claimed.active_child_claims(), 1);
3497 assert_eq!(claimed.domains()[0].used().get(), 3);
3498 assert_eq!(claimed.domains()[1].used().get(), 1);
3499 let state = coordinator.inner.state.lock().unwrap();
3500 assert_eq!(
3501 state.live_sequences[first.sequence().sparse_id() as usize]
3502 .unwrap()
3503 .active_child_claims,
3504 1
3505 );
3506 assert_eq!(
3507 state.live_sequences[second.sequence().sparse_id() as usize]
3508 .unwrap()
3509 .active_child_claims,
3510 1
3511 );
3512 drop(state);
3513
3514 drop(batch);
3515 let released = coordinator.snapshot().unwrap();
3516 assert_eq!(released.active_child_claims(), 0);
3517 assert_eq!(released.domains()[0].used().get(), 0);
3518 assert_eq!(released.domains()[1].used().get(), 0);
3519 assert_eq!(released.release_epoch(), before.release_epoch() + 1);
3520 drop(first);
3521 drop(second);
3522 drop(first_request);
3523 drop(second_request);
3524 }
3525
3526 #[test]
3527 fn batch_child_rejects_duplicate_and_foreign_parents_atomically() {
3528 let local = coordinator(2);
3529 let local_request = request(&local);
3530 let first = admit_sequence(&local, &local_request, &empty_demand());
3531 let before = local.snapshot().unwrap();
3532 assert!(local
3533 .try_claim_for_sequences(&[&first, &first], &demand(&[(1, 1)], &[(1, 1)]))
3534 .is_err());
3535 let after_duplicate = local.snapshot().unwrap();
3536 assert_eq!(before.domains, after_duplicate.domains);
3537 assert_eq!(after_duplicate.active_child_claims(), 0);
3538
3539 let foreign = coordinator(1);
3540 let foreign_request = request(&foreign);
3541 let foreign_sequence = admit_sequence(&foreign, &foreign_request, &empty_demand());
3542 assert!(matches!(
3543 local.try_claim_for_sequences(
3544 &[&first, &foreign_sequence],
3545 &demand(&[(1, 1)], &[(1, 1)])
3546 ),
3547 Err(VNextError::DynamicAdmissionContract {
3548 kind: DynamicAdmissionFaultKind::ForeignCoordinator,
3549 ..
3550 })
3551 ));
3552 let after_foreign = local.snapshot().unwrap();
3553 assert_eq!(before.domains, after_foreign.domains);
3554 assert_eq!(after_foreign.active_child_claims(), 0);
3555 }
3556
3557 #[test]
3558 fn batch_child_defer_and_reject_have_zero_partial_parent_effect() {
3559 let coordinator = coordinator(2);
3560 let request = request(&coordinator);
3561 let first = admit_sequence(&coordinator, &request, &demand(&[(2, 3)], &[(2, 3)]));
3562 let second = admit_sequence(&coordinator, &request, &empty_demand());
3563 let before = coordinator.snapshot().unwrap();
3564
3565 let deferred = coordinator
3566 .try_claim_for_sequences(
3567 &[&first, &second],
3568 &demand(&[(1, 1), (2, 2)], &[(1, 1), (2, 2)]),
3569 )
3570 .unwrap();
3571 assert!(matches!(deferred, BatchCapacityClaimDecision::Deferred(_)));
3572 let rejected = coordinator
3573 .try_claim_for_sequences(&[&first, &second], &demand(&[(1, 21)], &[(1, 21)]))
3574 .unwrap();
3575 assert!(matches!(
3576 rejected,
3577 BatchCapacityClaimDecision::PermanentRejected(_)
3578 ));
3579 let after = coordinator.snapshot().unwrap();
3580 assert_eq!(before.domains, after.domains);
3581 assert_eq!(after.active_child_claims(), 0);
3582 let state = coordinator.inner.state.lock().unwrap();
3583 assert_eq!(
3584 state.live_sequences[first.sequence().sparse_id() as usize]
3585 .unwrap()
3586 .active_child_claims,
3587 0
3588 );
3589 assert_eq!(
3590 state.live_sequences[second.sequence().sparse_id() as usize]
3591 .unwrap()
3592 .active_child_claims,
3593 0
3594 );
3595 }
3596
3597 #[test]
3598 fn overlapping_batch_children_track_each_parent_without_double_charging() {
3599 let coordinator = coordinator(3);
3600 let request = request(&coordinator);
3601 let first = admit_sequence(&coordinator, &request, &empty_demand());
3602 let second = admit_sequence(&coordinator, &request, &empty_demand());
3603 let third = admit_sequence(&coordinator, &request, &empty_demand());
3604 let first_batch = claimed_batch(
3605 coordinator
3606 .try_claim_for_sequences(&[&first, &second], &demand(&[(1, 1)], &[(1, 1)]))
3607 .unwrap(),
3608 );
3609 let second_batch = claimed_batch(
3610 coordinator
3611 .try_claim_for_sequences(&[&second, &third], &demand(&[(1, 1)], &[(1, 1)]))
3612 .unwrap(),
3613 );
3614 let snapshot = coordinator.snapshot().unwrap();
3615 assert_eq!(snapshot.active_child_claims(), 2);
3616 assert_eq!(snapshot.domains()[0].used().get(), 2);
3617 let state = coordinator.inner.state.lock().unwrap();
3618 let child_counts = [&first, &second, &third].map(|parent| {
3619 state.live_sequences[parent.sequence().sparse_id() as usize]
3620 .unwrap()
3621 .active_child_claims
3622 });
3623 assert_eq!(child_counts, [1, 2, 1]);
3624 drop(state);
3625 drop(first_batch);
3626 assert_eq!(coordinator.snapshot().unwrap().active_child_claims(), 1);
3627 drop(second_batch);
3628 let released = coordinator.snapshot().unwrap();
3629 assert_eq!(released.active_child_claims(), 0);
3630 assert_eq!(released.domains()[0].used().get(), 0);
3631 }
3632
3633 #[test]
3634 fn batch_child_unwind_releases_all_parent_edges() {
3635 let coordinator = coordinator(2);
3636 let request = request(&coordinator);
3637 let first = admit_sequence(&coordinator, &request, &empty_demand());
3638 let second = admit_sequence(&coordinator, &request, &empty_demand());
3639 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
3640 let _batch = claimed_batch(
3641 coordinator
3642 .try_claim_for_sequences(&[&first, &second], &demand(&[(1, 2)], &[(1, 2)]))
3643 .unwrap(),
3644 );
3645 panic!("inject batch invocation cancellation unwind");
3646 }));
3647 assert!(result.is_err());
3648 let snapshot = coordinator.snapshot().unwrap();
3649 assert!(!snapshot.poisoned());
3650 assert_eq!(snapshot.active_child_claims(), 0);
3651 assert_eq!(snapshot.domains()[0].used().get(), 0);
3652 let state = coordinator.inner.state.lock().unwrap();
3653 assert!([&first, &second].iter().all(|parent| state.live_sequences
3654 [parent.sequence().sparse_id() as usize]
3655 .unwrap()
3656 .active_child_claims
3657 == 0));
3658 }
3659
3660 #[test]
3661 fn batch_child_rejects_stale_later_parent_without_partial_effect() {
3662 let coordinator = coordinator(2);
3663 let request = request(&coordinator);
3664 let first = admit_sequence(&coordinator, &request, &empty_demand());
3665 let second = admit_sequence(&coordinator, &request, &empty_demand());
3666 {
3667 let mut state = coordinator.inner.state.lock().unwrap();
3668 state.live_sequences[second.sequence().sparse_id() as usize]
3669 .as_mut()
3670 .unwrap()
3671 .generation += 1;
3672 }
3673 let before = coordinator.snapshot().unwrap();
3674 assert!(coordinator
3675 .try_claim_for_sequences(&[&first, &second], &demand(&[(1, 2)], &[(1, 2)]))
3676 .is_err());
3677 let after = coordinator.snapshot().unwrap();
3678 assert_eq!(before.domains, after.domains);
3679 assert_eq!(after.active_child_claims(), 0);
3680 {
3681 let mut state = coordinator.inner.state.lock().unwrap();
3682 state.live_sequences[second.sequence().sparse_id() as usize]
3683 .as_mut()
3684 .unwrap()
3685 .generation = second.sequence().generation();
3686 }
3687 }
3688
3689 #[test]
3690 fn early_release_of_any_batch_parent_fails_closed_and_retains_shared_claim() {
3691 let coordinator = coordinator(2);
3692 let request = request(&coordinator);
3693 let first = admit_sequence(&coordinator, &request, &empty_demand());
3694 let mut second = admit_sequence(&coordinator, &request, &empty_demand());
3695 let batch = claimed_batch(
3696 coordinator
3697 .try_claim_for_sequences(&[&first, &second], &demand(&[(1, 2)], &[(1, 2)]))
3698 .unwrap(),
3699 );
3700 assert!(!second.release_inner());
3701 let snapshot = coordinator.snapshot().unwrap();
3702 assert!(snapshot.poisoned());
3703 assert_eq!(snapshot.active_sequences(), 2);
3704 assert_eq!(snapshot.active_child_claims(), 1);
3705 assert_eq!(snapshot.domains()[0].used().get(), 2);
3706 drop(batch);
3707 drop(first);
3708 drop(second);
3709 drop(request);
3710 }
3711
3712 #[test]
3713 fn child_multi_domain_defer_and_reject_have_zero_partial_effect() {
3714 let coordinator = coordinator(2);
3715 let request = request(&coordinator);
3716 let parent = admit_sequence(
3717 &coordinator,
3718 &request,
3719 &demand(&[(1, 2), (2, 3)], &[(1, 2), (2, 3)]),
3720 );
3721 let before = coordinator.snapshot().unwrap();
3722 let deferred = coordinator
3723 .try_claim_for_sequence(&parent, &demand(&[(1, 2), (2, 2)], &[(1, 2), (2, 2)]))
3724 .unwrap();
3725 assert!(matches!(deferred, CapacityClaimDecision::Deferred(_)));
3726 let after_defer = coordinator.snapshot().unwrap();
3727 assert_eq!(before.domains, after_defer.domains);
3728 assert_eq!(after_defer.active_child_claims(), 0);
3729
3730 let rejected = coordinator
3731 .try_claim_for_sequence(&parent, &demand(&[(1, 21), (2, 1)], &[(1, 21), (2, 1)]))
3732 .unwrap();
3733 assert!(matches!(
3734 rejected,
3735 CapacityClaimDecision::PermanentRejected(_)
3736 ));
3737 let after_reject = coordinator.snapshot().unwrap();
3738 assert_eq!(before.domains, after_reject.domains);
3739 assert_eq!(after_reject.active_child_claims(), 0);
3740 }
3741
3742 #[test]
3743 fn concurrent_children_preserve_global_and_per_parent_counts() {
3744 const WORKER_COUNT: usize = 3;
3745 const _: () = assert!(WORKER_COUNT <= TEST_NATIVE_WORKER_LIMIT);
3746
3747 let coordinator = coordinator(2);
3748 let request = Arc::new(request(&coordinator));
3749 let first = Arc::new(admit_sequence(
3750 &coordinator,
3751 &request,
3752 &demand(&[(1, 1)], &[(1, 1)]),
3753 ));
3754 let second = Arc::new(admit_sequence(
3755 &coordinator,
3756 &request,
3757 &demand(&[(1, 1)], &[(1, 1)]),
3758 ));
3759 let gate = CancellableTestGate::new(WORKER_COUNT);
3760 thread::scope(|scope| {
3761 let mut cancel_on_unwind = CancelTestGateOnDrop::new(&gate);
3762 let parents = [Arc::clone(&first), Arc::clone(&first), Arc::clone(&second)];
3763 let mut handles: Vec<thread::ScopedJoinHandle<'_, ()>> =
3764 Vec::with_capacity(WORKER_COUNT);
3765 for (worker_index, parent) in parents.into_iter().enumerate() {
3766 let worker_gate = Arc::clone(&gate);
3767 let coordinator = &coordinator;
3768 let handle = match thread::Builder::new()
3769 .name(format!("admission-child-{worker_index}"))
3770 .spawn_scoped(scope, move || {
3771 cancel_gate_on_unwind(&worker_gate, || {
3772 let child = claimed_child(
3773 coordinator
3774 .try_claim_for_sequence(&parent, &demand(&[(1, 1)], &[(1, 1)]))
3775 .unwrap(),
3776 );
3777 let _ = worker_gate.arrive_and_wait();
3778 drop(child);
3779 });
3780 }) {
3781 Ok(handle) => handle,
3782 Err(error) => {
3783 gate.cancel();
3784 for handle in handles.drain(..) {
3785 let _ = handle.join();
3786 }
3787 panic!("failed to spawn bounded child worker: {error}");
3788 }
3789 };
3790 handles.push(handle);
3791 }
3792 if !gate.wait_until_ready(WORKER_COUNT) {
3793 gate.cancel();
3794 for handle in handles.drain(..) {
3795 let _ = handle.join();
3796 }
3797 panic!("bounded child worker cancelled before every worker became ready");
3798 }
3799 let snapshot = coordinator.snapshot().unwrap();
3800 assert_eq!(snapshot.active_requests(), 1);
3801 assert_eq!(snapshot.active_sequences(), 2);
3802 assert_eq!(snapshot.active_child_claims(), 3);
3803 let state = coordinator.inner.state.lock().unwrap();
3804 assert_eq!(
3805 state.live_sequences[first.sequence().sparse_id() as usize]
3806 .unwrap()
3807 .active_child_claims,
3808 2
3809 );
3810 assert_eq!(
3811 state.live_sequences[second.sequence().sparse_id() as usize]
3812 .unwrap()
3813 .active_child_claims,
3814 1
3815 );
3816 drop(state);
3817 gate.release();
3818 cancel_on_unwind.disarm();
3819 for handle in handles {
3820 handle.join().unwrap();
3821 }
3822 });
3823 assert_eq!(coordinator.snapshot().unwrap().active_child_claims(), 0);
3824 }
3825
3826 #[test]
3827 fn child_unwind_releases_without_poisoning_parent() {
3828 let coordinator = coordinator(1);
3829 let request = request(&coordinator);
3830 let parent = admit_sequence(&coordinator, &request, &demand(&[(1, 1)], &[(1, 1)]));
3831 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
3832 let _child = claimed_child(
3833 coordinator
3834 .try_claim_for_sequence(&parent, &demand(&[(1, 2)], &[(1, 2)]))
3835 .unwrap(),
3836 );
3837 panic!("inject invocation cancellation unwind");
3838 }));
3839 assert!(result.is_err());
3840 let snapshot = coordinator.snapshot().unwrap();
3841 assert!(!snapshot.poisoned());
3842 assert_eq!(snapshot.active_sequences(), 1);
3843 assert_eq!(snapshot.active_child_claims(), 0);
3844 assert_eq!(snapshot.domains()[0].used().get(), 1);
3845 }
3846
3847 #[test]
3848 fn child_claim_rejects_stale_sequence_generation_without_side_effect() {
3849 let coordinator = coordinator(1);
3850 let request = request(&coordinator);
3851 let parent = admit_sequence(&coordinator, &request, &demand(&[(1, 1)], &[(1, 1)]));
3852 {
3853 let mut state = coordinator.inner.state.lock().unwrap();
3854 state.live_sequences[parent.sequence().sparse_id() as usize]
3855 .as_mut()
3856 .unwrap()
3857 .generation += 1;
3858 }
3859 let before = coordinator.snapshot().unwrap();
3860 assert!(coordinator
3861 .try_claim_for_sequence(&parent, &demand(&[(1, 1)], &[(1, 1)]))
3862 .is_err());
3863 let after = coordinator.snapshot().unwrap();
3864 assert_eq!(before.domains, after.domains);
3865 assert_eq!(after.active_child_claims(), 0);
3866 {
3867 let mut state = coordinator.inner.state.lock().unwrap();
3868 state.live_sequences[parent.sequence().sparse_id() as usize]
3869 .as_mut()
3870 .unwrap()
3871 .generation = parent.sequence().generation();
3872 }
3873 }
3874
3875 #[test]
3876 fn child_claim_is_counted_in_future_request_epoch_headroom() {
3877 let coordinator = coordinator(1);
3878 let request = request(&coordinator);
3879 let parent = admit_sequence(&coordinator, &request, &demand(&[(1, 1)], &[(1, 1)]));
3880 let child = claimed_child(
3881 coordinator
3882 .try_claim_for_sequence(&parent, &demand(&[(1, 1)], &[(1, 1)]))
3883 .unwrap(),
3884 );
3885 {
3886 let mut state = coordinator.inner.state.lock().unwrap();
3887 state.release_epoch = u64::MAX - 3;
3888 }
3889 let before = coordinator.snapshot().unwrap();
3890 assert!(matches!(
3891 coordinator.try_admit_request(&empty_demand()),
3892 Err(VNextError::DynamicAdmissionContract {
3893 kind: DynamicAdmissionFaultKind::EpochExhausted,
3894 ..
3895 })
3896 ));
3897 let after = coordinator.snapshot().unwrap();
3898 assert_eq!(before.active_requests(), after.active_requests());
3899 assert_eq!(before.active_sequences(), after.active_sequences());
3900 assert_eq!(before.active_child_claims(), after.active_child_claims());
3901 assert_eq!(before.domains, after.domains);
3902 drop(child);
3903 drop(parent);
3904 drop(request);
3905 assert_eq!(coordinator.epochs().unwrap().release_epoch(), u64::MAX);
3906 }
3907
3908 #[test]
3909 fn early_sequence_release_with_active_child_fails_closed() {
3910 let coordinator = coordinator(1);
3911 let request = request(&coordinator);
3912 let mut parent = admit_sequence(&coordinator, &request, &demand(&[(1, 1)], &[(1, 1)]));
3913 let child = claimed_child(
3914 coordinator
3915 .try_claim_for_sequence(&parent, &demand(&[(1, 1)], &[(1, 1)]))
3916 .unwrap(),
3917 );
3918 assert!(!parent.release_inner());
3919 let snapshot = coordinator.snapshot().unwrap();
3920 assert!(snapshot.poisoned());
3921 assert_eq!(snapshot.active_sequences(), 1);
3922 assert_eq!(snapshot.active_child_claims(), 1);
3923 drop(child);
3924 drop(parent);
3925 drop(request);
3926 }
3927
3928 #[test]
3929 fn poisoned_child_drop_retains_capacity_and_child_counts() {
3930 let coordinator = coordinator(1);
3931 let request = request(&coordinator);
3932 let parent = admit_sequence(&coordinator, &request, &demand(&[(1, 1)], &[(1, 1)]));
3933 let child = claimed_child(
3934 coordinator
3935 .try_claim_for_sequence(&parent, &demand(&[(1, 2)], &[(1, 2)]))
3936 .unwrap(),
3937 );
3938 let inner = Arc::clone(&coordinator.inner);
3939 let _ = thread::spawn(move || {
3940 let _guard = inner.state.lock().unwrap();
3941 panic!("poison coordinator before child release");
3942 })
3943 .join();
3944 drop(child);
3945 let snapshot = coordinator.snapshot().unwrap();
3946 assert!(snapshot.poisoned());
3947 assert_eq!(snapshot.active_requests(), 1);
3948 assert_eq!(snapshot.active_sequences(), 1);
3949 assert_eq!(snapshot.active_child_claims(), 1);
3950 assert_eq!(snapshot.domains()[0].used().get(), 3);
3951 drop(parent);
3952 drop(request);
3953 }
3954
3955 #[test]
3956 fn child_claim_rejects_foreign_parent_and_epoch_exhaustion_atomically() {
3957 let local = coordinator(2);
3958 let foreign = coordinator(2);
3959 let foreign_request = request(&foreign);
3960 let foreign_parent = admit_sequence(
3961 &foreign,
3962 &foreign_request,
3963 &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)]),
3964 );
3965 assert!(matches!(
3966 local.try_claim_for_sequence(
3967 &foreign_parent,
3968 &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)])
3969 ),
3970 Err(VNextError::DynamicAdmissionContract {
3971 kind: DynamicAdmissionFaultKind::ForeignCoordinator,
3972 ..
3973 })
3974 ));
3975
3976 let local_request = request(&local);
3977 let parent = admit_sequence(
3978 &local,
3979 &local_request,
3980 &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)]),
3981 );
3982 {
3983 let mut state = local.inner.state.lock().unwrap();
3984 state.release_epoch = u64::MAX - 1;
3985 }
3986 let before = local.snapshot().unwrap();
3987 assert!(matches!(
3988 local.try_claim_for_sequence(&parent, &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)])),
3989 Err(VNextError::DynamicAdmissionContract {
3990 kind: DynamicAdmissionFaultKind::EpochExhausted,
3991 ..
3992 })
3993 ));
3994 let after = local.snapshot().unwrap();
3995 assert_eq!(before.domains, after.domains);
3996 assert_eq!(after.active_child_claims(), 0);
3997 }
3998
3999 #[test]
4000 fn multi_domain_failure_has_zero_partial_effect() {
4001 let coordinator = coordinator(8);
4002 let request = request(&coordinator);
4003 let before = coordinator.snapshot().unwrap();
4004 let decision = coordinator
4005 .try_admit_sequence_for_request(&request, &demand(&[(1, 3), (2, 4)], &[(1, 3), (2, 5)]))
4006 .unwrap();
4007 assert!(matches!(decision, AdmissionDecision::PermanentRejected(_)));
4008 let after = coordinator.snapshot().unwrap();
4009 assert_eq!(before.domains, after.domains);
4010 assert_eq!(after.active_sequences(), 0);
4011 assert_eq!(after.live_sequence_records(), 0);
4012 }
4013
4014 #[test]
4015 fn temporary_multi_domain_shortfall_has_zero_partial_effect() {
4016 let coordinator = coordinator(8);
4017 let request = request(&coordinator);
4018 let held = admit_sequence(
4019 &coordinator,
4020 &request,
4021 &demand(&[(1, 1), (2, 3)], &[(1, 1), (2, 3)]),
4022 );
4023 let before = coordinator.snapshot().unwrap();
4024 let decision = coordinator
4025 .try_admit_sequence_for_request(&request, &demand(&[(1, 2), (2, 2)], &[(1, 2), (2, 2)]))
4026 .unwrap();
4027 assert!(matches!(decision, AdmissionDecision::Deferred(_)));
4028 let after = coordinator.snapshot().unwrap();
4029 assert_eq!(before.domains, after.domains);
4030 assert_eq!(before.active_sequences(), after.active_sequences());
4031 drop(held);
4032 }
4033
4034 #[test]
4035 fn growth_defer_and_permanent_reject_are_distinct() {
4036 let coordinator = coordinator(8);
4037 let request = request(&coordinator);
4038 let growth = coordinator
4039 .try_admit_sequence_for_request(
4040 &request,
4041 &demand(&[(1, 11), (2, 1)], &[(1, 11), (2, 1)]),
4042 )
4043 .unwrap();
4044 assert!(matches!(
4045 growth,
4046 AdmissionDecision::Deferred(AdmissionDeferred {
4047 action: DeferredAction::AwaitBackingGrowth,
4048 ..
4049 })
4050 ));
4051 let impossible = coordinator
4052 .try_admit_sequence_for_request(
4053 &request,
4054 &demand(&[(1, 21), (2, 1)], &[(1, 21), (2, 1)]),
4055 )
4056 .unwrap();
4057 assert!(matches!(
4058 impossible,
4059 AdmissionDecision::PermanentRejected(_)
4060 ));
4061 assert!(matches!(
4062 coordinator
4063 .try_admit_sequence_for_request(
4064 &request,
4065 &demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]),
4066 )
4067 .unwrap(),
4068 AdmissionDecision::Admitted(_)
4069 ));
4070 }
4071
4072 #[test]
4073 fn lease_authority_is_bound_to_the_exact_coordinator() {
4074 let coordinator_a = coordinator(8);
4075 let coordinator_b = coordinator(8);
4076 assert_ne!(coordinator_a.id(), coordinator_b.id());
4077 let request = request(&coordinator_a);
4078 let lease = admit_sequence(
4079 &coordinator_a,
4080 &request,
4081 &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)]),
4082 );
4083 assert!(coordinator_a.owns(&lease));
4084 assert!(!coordinator_b.owns(&lease));
4085 }
4086
4087 #[test]
4088 fn sparse_sequence_ids_reuse_storage_and_change_generation() {
4089 let coordinator = coordinator(u32::MAX);
4090 let parent_request = request(&coordinator);
4091 let sequence_demand = demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)]);
4092 let mut previous_generation = 0;
4093 for _ in 0..1_000_000 {
4094 let lease = admit_sequence(&coordinator, &parent_request, &sequence_demand);
4095 assert_eq!(lease.sequence().sparse_id(), 0);
4096 assert!(lease.sequence().generation() > previous_generation);
4097 previous_generation = lease.sequence().generation();
4098 drop(lease);
4099 }
4100 let snapshot = coordinator.snapshot().unwrap();
4101 assert_eq!(snapshot.active_sequences(), 0);
4102 assert_eq!(snapshot.live_sequence_records(), 0);
4103 assert_eq!(snapshot.reusable_sequence_ids(), 1);
4104 assert_eq!(snapshot.release_epoch(), 1_000_001);
4105 assert_eq!(snapshot.capacity_epoch(), 1);
4106 }
4107
4108 #[test]
4109 fn lease_release_uses_preallocated_reuse_storage() {
4110 let coordinator = coordinator(1);
4111 let request = request(&coordinator);
4112 let lease = admit_sequence(
4113 &coordinator,
4114 &request,
4115 &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)]),
4116 );
4117 let before = {
4118 let state = coordinator.inner.state.lock().unwrap();
4119 assert!(state.reusable_sequence_ids.capacity() >= state.live_sequences.len());
4120 (
4121 state.live_sequences.capacity(),
4122 state.reusable_sequence_ids.capacity(),
4123 )
4124 };
4125 drop(lease);
4126 let state = coordinator.inner.state.lock().unwrap();
4127 assert_eq!(state.live_sequences.capacity(), before.0);
4128 assert_eq!(state.reusable_sequence_ids.capacity(), before.1);
4129 assert_eq!(state.reusable_sequence_ids.as_slice(), &[0]);
4130 }
4131
4132 #[test]
4133 fn snapshot_counts_live_authorities_independently_from_active_counter() {
4134 let coordinator = coordinator(1);
4135 let mut state = coordinator.inner.state.lock().unwrap();
4136 state.active_sequences = 1;
4137 state.poisoned = true;
4138 let snapshot = state.snapshot(coordinator.id());
4139 assert_eq!(snapshot.active_sequences(), 1);
4140 assert_eq!(snapshot.live_sequence_records(), 0);
4141 }
4142
4143 #[test]
4144 fn concurrent_admission_never_exceeds_ceiling() {
4145 const SEQUENCE_CEILING: u32 = 4;
4146 const WORKER_COUNT: usize = 8;
4147 const _: () = assert!(WORKER_COUNT <= TEST_NATIVE_WORKER_LIMIT);
4148
4149 let coordinator = Arc::new(coordinator(SEQUENCE_CEILING));
4150 let parent_request = Arc::new(request(&coordinator));
4151 let gate = CancellableTestGate::new(WORKER_COUNT);
4152 let mut handles: Vec<thread::JoinHandle<Option<AdmissionDecision>>> =
4153 Vec::with_capacity(WORKER_COUNT);
4154 for worker_index in 0..WORKER_COUNT {
4155 let coordinator = Arc::clone(&coordinator);
4156 let parent_request = Arc::clone(&parent_request);
4157 let worker_gate = Arc::clone(&gate);
4158 let handle = match thread::Builder::new()
4159 .name(format!("admission-sequence-{worker_index}"))
4160 .spawn(move || {
4161 cancel_gate_on_unwind(&worker_gate, || {
4162 if !worker_gate.arrive_and_wait() {
4163 return None;
4164 }
4165 Some(
4166 coordinator
4167 .try_admit_sequence_for_request(&parent_request, &empty_demand())
4168 .unwrap(),
4169 )
4170 })
4171 }) {
4172 Ok(handle) => handle,
4173 Err(error) => {
4174 gate.cancel();
4175 for handle in handles.drain(..) {
4176 let _ = handle.join();
4177 }
4178 panic!("failed to spawn bounded admission worker: {error}");
4179 }
4180 };
4181 handles.push(handle);
4182 }
4183 if !gate.wait_until_ready(WORKER_COUNT) {
4184 gate.cancel();
4185 for handle in handles.drain(..) {
4186 let _ = handle.join();
4187 }
4188 panic!("bounded admission worker cancelled before every worker became ready");
4189 }
4190 gate.release();
4191 let mut leases = Vec::new();
4192 for handle in handles {
4193 if let Some(AdmissionDecision::Admitted(lease)) = handle.join().unwrap() {
4194 leases.push(lease);
4195 }
4196 }
4197 assert_eq!(leases.len(), SEQUENCE_CEILING as usize);
4198 assert_eq!(
4199 coordinator.snapshot().unwrap().active_sequences(),
4200 SEQUENCE_CEILING
4201 );
4202
4203 let preempt_demand = AdmissionDemand::from_plan(
4204 CapacityVector::empty(),
4205 CapacityVector::empty(),
4206 AdmissionFitPolicy::ImmediateOnly,
4207 AdmissionPressureAction::PreemptAndRecompute,
4208 )
4209 .unwrap();
4210 let preflight = coordinator
4211 .preflight_sequence_ceiling_for_request(&parent_request, &preempt_demand)
4212 .unwrap();
4213 match preflight {
4214 AdmissionPreflightDecision::Deferred(deferred) => {
4215 assert_eq!(deferred.action(), DeferredAction::PreemptAndRecompute);
4216 assert!(deferred
4217 .blockers()
4218 .iter()
4219 .any(|blocker| blocker.kind() == CapacityShortfallKind::ActiveSequenceCeiling));
4220 }
4221 _ => panic!("full sequence ceiling must defer preflight"),
4222 }
4223
4224 for (pressure_action, expected) in [
4225 (
4226 AdmissionPressureAction::WaitForRelease,
4227 DeferredAction::WaitForRelease,
4228 ),
4229 (
4230 AdmissionPressureAction::PreemptAndRecompute,
4231 DeferredAction::PreemptAndRecompute,
4232 ),
4233 ] {
4234 let growth_demand = AdmissionDemand::from_plan(
4235 vector(&[(1, 11)]),
4236 vector(&[(1, 11)]),
4237 AdmissionFitPolicy::ImmediateOnly,
4238 pressure_action,
4239 )
4240 .unwrap();
4241 let decision = coordinator
4242 .preflight_sequence_ceiling_for_request(&parent_request, &growth_demand)
4243 .unwrap();
4244 match decision {
4245 AdmissionPreflightDecision::Deferred(deferred) => {
4246 assert_eq!(deferred.action(), expected);
4247 assert!(deferred.blockers().iter().any(|blocker| {
4248 blocker.kind() == CapacityShortfallKind::ActiveSequenceCeiling
4249 }));
4250 assert!(deferred.blockers().iter().any(|blocker| {
4251 blocker.kind() == CapacityShortfallKind::BackingGrowthRequired
4252 }));
4253 }
4254 _ => panic!("mixed ceiling and growth pressure must remain a logical defer"),
4255 }
4256 }
4257 drop(leases);
4258 assert_eq!(coordinator.snapshot().unwrap().active_sequences(), 0);
4259 }
4260
4261 #[test]
4262 fn zero_domain_plan_still_uses_dynamic_sequence_authority() {
4263 let coordinator = LogicalAdmissionCoordinator::new(Vec::new(), u32::MAX).unwrap();
4264 let parent_request = request(&coordinator);
4265 let sequence_demand = empty_demand();
4266 let lease = admit_sequence(&coordinator, &parent_request, &sequence_demand);
4267 assert!(lease.claims().is_empty());
4268 assert_eq!(coordinator.snapshot().unwrap().active_sequences(), 1);
4269 drop(lease);
4270 assert_eq!(coordinator.snapshot().unwrap().active_sequences(), 0);
4271 }
4272
4273 #[test]
4274 fn sequence_issue_failure_has_zero_side_effect() {
4275 let coordinator = coordinator(8);
4276 let request = request(&coordinator);
4277 {
4278 let mut state = coordinator.inner.state.lock().unwrap();
4279 state.next_sequence_generation = u64::MAX;
4280 }
4281 let before = coordinator.snapshot().unwrap();
4282 assert!(
4283 coordinator
4284 .try_admit_sequence_for_request(
4285 &request,
4286 &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)]),
4287 )
4288 .is_err()
4289 );
4290 let after = coordinator.snapshot().unwrap();
4291 assert_eq!(before.domains, after.domains);
4292 assert_eq!(after.active_sequences(), 0);
4293 assert_eq!(after.live_sequence_records(), 0);
4294 }
4295
4296 #[test]
4297 fn epoch_exhaustion_rejects_admission_before_claim() {
4298 let coordinator = coordinator(8);
4299 let request = request(&coordinator);
4300 {
4301 let mut state = coordinator.inner.state.lock().unwrap();
4302 state.release_epoch = u64::MAX;
4303 }
4304 let before = coordinator.snapshot().unwrap();
4305 assert!(
4306 coordinator
4307 .try_admit_sequence_for_request(
4308 &request,
4309 &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)]),
4310 )
4311 .is_err()
4312 );
4313 let after = coordinator.snapshot().unwrap();
4314 assert_eq!(before.domains, after.domains);
4315 assert_eq!(after.active_sequences(), 0);
4316 }
4317
4318 #[test]
4319 fn waiter_recheck_closes_release_and_growth_races() {
4320 let coordinator = coordinator(1);
4321 let request = request(&coordinator);
4322 let observed = wait_for_domains(&coordinator, &[1]);
4323 let registration = coordinator.register_waiter(observed).unwrap();
4324 assert!(!registration.recheck().unwrap().should_retry());
4325
4326 coordinator
4327 .set_domain_total(domain(1), CapacityUnits::new(11))
4328 .unwrap();
4329 let recheck = registration.recheck().unwrap();
4330 assert!(recheck.changed_since_registration());
4331 assert_eq!(recheck.current().capacity_epoch(), 2);
4332
4333 let lease = admit_sequence(
4334 &coordinator,
4335 &request,
4336 &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)]),
4337 );
4338 let before_release = wait_for_domains(&coordinator, &[1]);
4339 drop(lease);
4340 let late_registration = coordinator.register_waiter(before_release).unwrap();
4341 assert!(late_registration
4342 .recheck()
4343 .unwrap()
4344 .changed_since_observation());
4345 assert_eq!(coordinator.epochs().unwrap().capacity_epoch(), 2);
4346 }
4347
4348 #[test]
4349 fn waiter_retries_only_when_its_exact_domain_changes() {
4350 let coordinator = coordinator(8);
4351 let observed = wait_for_domains(&coordinator, &[1]);
4352 let registration = coordinator.register_waiter(observed).unwrap();
4353
4354 let before = coordinator.epochs().unwrap();
4355 coordinator
4356 .notify_domain_availability_changed(domain(2))
4357 .unwrap();
4358 let unrelated = registration.recheck().unwrap();
4359 assert_eq!(
4360 unrelated.current().capacity_epoch(),
4361 before.capacity_epoch() + 1
4362 );
4363 assert!(!unrelated.changed_since_observation());
4364 assert!(!unrelated.changed_since_registration());
4365 assert!(!unrelated.should_retry());
4366
4367 coordinator
4368 .notify_domain_availability_changed(domain(1))
4369 .unwrap();
4370 let relevant = registration.recheck().unwrap();
4371 assert!(relevant.changed_since_observation());
4372 assert!(relevant.changed_since_registration());
4373 assert!(relevant.should_retry());
4374 }
4375
4376 #[test]
4377 fn wait_snapshot_keeps_pre_mutation_audit_and_exact_source_together() {
4378 let coordinator = coordinator(8);
4379 let snapshot = coordinator
4380 .wait_snapshot_for_domains([domain(1), domain(2)])
4381 .unwrap()
4382 .narrow_to_domains([domain(1)])
4383 .unwrap();
4384 let observed_epochs = snapshot.epochs();
4385
4386 coordinator
4387 .notify_domain_availability_changed(domain(1))
4388 .unwrap();
4389 let mut current = Vec::new();
4390 let current_epochs = coordinator.write_availability_epochs(&mut current).unwrap();
4391
4392 assert_eq!(
4393 snapshot.wait_condition().observed(),
4394 &[
4395 CapacityAvailabilityEpoch::new(CapacityAvailabilitySource::Domain(domain(1)), 1,)
4396 .unwrap()
4397 ]
4398 );
4399 assert_eq!(observed_epochs.capacity_epoch(), 1);
4400 assert_eq!(current_epochs.capacity_epoch(), 2);
4401 assert!(snapshot.wait_condition().changed_since(¤t).unwrap());
4402 }
4403
4404 #[test]
4405 fn active_sequence_waiter_retries_when_a_slot_is_released() {
4406 let coordinator = coordinator(1);
4407 let first_request = request(&coordinator);
4408 let first = admit_sequence(&coordinator, &first_request, &empty_demand());
4409 let second_request = request(&coordinator);
4410 let deferred = match coordinator
4411 .try_admit_sequence_for_request(&second_request, &empty_demand())
4412 .unwrap()
4413 {
4414 AdmissionDecision::Deferred(deferred) => deferred,
4415 _ => panic!("active-sequence ceiling must defer the second sequence"),
4416 };
4417 assert_eq!(
4418 deferred.wait_condition().observed()[0].source(),
4419 CapacityAvailabilitySource::ActiveSequenceSlots
4420 );
4421 let registration = coordinator
4422 .register_waiter(deferred.wait_condition().clone())
4423 .unwrap();
4424 assert!(!registration.recheck().unwrap().should_retry());
4425
4426 drop(first);
4427 assert!(registration.recheck().unwrap().should_retry());
4428 }
4429
4430 #[test]
4431 fn changed_source_cannot_hide_another_source_regression() {
4432 let first = CapacityAvailabilitySource::Domain(domain(1));
4433 let second = CapacityAvailabilitySource::Domain(domain(2));
4434 let observed = CapacityWaitCondition::from_observation(
4435 41,
4436 vec![
4437 CapacityAvailabilityEpoch::new(first, 5).unwrap(),
4438 CapacityAvailabilityEpoch::new(second, 5).unwrap(),
4439 ],
4440 )
4441 .unwrap();
4442 let current = vec![
4443 CapacityAvailabilityEpoch::new(first, 6).unwrap(),
4444 CapacityAvailabilityEpoch::new(second, 4).unwrap(),
4445 ];
4446
4447 assert!(matches!(
4448 observed.changed_since(¤t),
4449 Err(VNextError::DynamicAdmissionContract {
4450 kind: DynamicAdmissionFaultKind::EpochRegression,
4451 ..
4452 })
4453 ));
4454 }
4455
4456 #[test]
4457 fn allocator_availability_change_advances_capacity_epoch_without_recounting_units() {
4458 let coordinator = coordinator(8);
4459 let before = coordinator.snapshot().unwrap();
4460 let observed = wait_for_domains(&coordinator, &[1]);
4461 let registration = coordinator.register_waiter(observed).unwrap();
4462
4463 let changed = coordinator
4464 .notify_domain_availability_changed(domain(1))
4465 .unwrap();
4466 let after = coordinator.snapshot().unwrap();
4467 assert_eq!(changed.capacity_epoch(), before.capacity_epoch() + 1);
4468 assert_eq!(changed.release_epoch(), before.release_epoch());
4469 assert_eq!(after.domains(), before.domains());
4470 assert!(registration.recheck().unwrap().should_retry());
4471
4472 let before_reject = coordinator.epochs().unwrap();
4473 assert!(coordinator
4474 .notify_domain_availability_changed(domain(99))
4475 .is_err());
4476 assert_eq!(coordinator.epochs().unwrap(), before_reject);
4477 }
4478
4479 #[test]
4480 fn multi_domain_growth_validates_all_before_one_epoch_commit() {
4481 let coordinator = coordinator(8);
4482 let before = coordinator.snapshot().unwrap();
4483 assert!(coordinator
4484 .set_domain_totals(&[
4485 (domain(1), CapacityUnits::new(11)),
4486 (domain(2), CapacityUnits::new(5)),
4487 ])
4488 .is_err());
4489 let rejected = coordinator.snapshot().unwrap();
4490 assert_eq!(before.domains, rejected.domains);
4491 assert_eq!(before.capacity_epoch(), rejected.capacity_epoch());
4492
4493 let committed = coordinator
4494 .set_domain_totals(&[
4495 (domain(1), CapacityUnits::new(11)),
4496 (domain(2), CapacityUnits::new(4)),
4497 ])
4498 .unwrap();
4499 assert_eq!(committed.capacity_epoch(), before.capacity_epoch() + 1);
4500 assert_eq!(
4501 coordinator.snapshot().unwrap().domains()[0].total().get(),
4502 11
4503 );
4504 }
4505
4506 #[tokio::test]
4507 async fn waiter_listener_closes_recheck_to_park_race() {
4508 let coordinator = coordinator(1);
4509 let request = request(&coordinator);
4510 let lease = admit_sequence(
4511 &coordinator,
4512 &request,
4513 &demand(&[(1, 1), (2, 1)], &[(1, 1), (2, 1)]),
4514 );
4515 let observed_epochs = coordinator.epochs().unwrap();
4516 let observed = wait_for_domains(&coordinator, &[1]);
4517 let registration = coordinator.register_waiter(observed).unwrap();
4518 assert!(!registration.recheck().unwrap().should_retry());
4519 drop(lease);
4520 let changed = tokio::time::timeout(
4521 std::time::Duration::from_secs(1),
4522 registration.wait_for_change(),
4523 )
4524 .await
4525 .expect("listener must not miss a release between recheck and park")
4526 .unwrap();
4527 assert!(changed.release_epoch() > observed_epochs.release_epoch());
4528 assert_eq!(changed.coordinator_id(), coordinator.id());
4529 }
4530
4531 #[tokio::test]
4532 async fn mutation_unwind_wakes_parked_waiter_with_terminal_error() {
4533 let coordinator = coordinator(1);
4534 let observed = wait_for_domains(&coordinator, &[1]);
4535 let registration = coordinator.register_waiter(observed).unwrap();
4536 let waiter = tokio::spawn(async move { registration.wait_for_change().await });
4537 tokio::task::yield_now().await;
4538
4539 let inner = Arc::clone(&coordinator.inner);
4540 let _ = thread::spawn(move || {
4541 let _mutation = inner.lock_mutation().unwrap();
4542 panic!("inject mutation unwind");
4543 })
4544 .join();
4545
4546 let result = tokio::time::timeout(std::time::Duration::from_secs(1), waiter)
4547 .await
4548 .expect("mutation unwind must wake an already parked waiter")
4549 .expect("waiter task must not panic");
4550 assert!(matches!(
4551 result,
4552 Err(VNextError::DynamicAdmissionContract {
4553 kind: DynamicAdmissionFaultKind::Poisoned,
4554 ..
4555 })
4556 ));
4557 assert!(coordinator.snapshot().unwrap().poisoned());
4558 }
4559
4560 #[test]
4561 fn sequence_unwind_releases_lease_without_poisoning_coordinator() {
4562 let coordinator = coordinator(1);
4563 let request = request(&coordinator);
4564 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
4565 let _lease = admit_sequence(
4566 &coordinator,
4567 &request,
4568 &demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]),
4569 );
4570 panic!("inject request panic outside coordinator mutation");
4571 }));
4572 assert!(result.is_err());
4573
4574 let snapshot = coordinator.snapshot().unwrap();
4575 assert!(!snapshot.poisoned());
4576 assert_eq!(snapshot.active_sequences(), 0);
4577 assert_eq!(snapshot.live_sequence_records(), 0);
4578 assert!(snapshot
4579 .domains()
4580 .iter()
4581 .all(|domain| domain.used().get() == 0));
4582
4583 let retry = coordinator
4584 .try_admit_sequence_for_request(&request, &demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]))
4585 .unwrap();
4586 assert!(matches!(retry, AdmissionDecision::Admitted(_)));
4587 }
4588
4589 #[test]
4590 fn request_unwind_releases_request_capacity_without_poisoning_coordinator() {
4591 let coordinator = coordinator(1);
4592 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
4593 let _request = admitted_request(
4594 coordinator
4595 .try_admit_request(&demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]))
4596 .unwrap(),
4597 );
4598 panic!("inject request scope unwind");
4599 }));
4600 assert!(result.is_err());
4601 let snapshot = coordinator.snapshot().unwrap();
4602 assert!(!snapshot.poisoned());
4603 assert_eq!(snapshot.active_requests(), 0);
4604 assert_eq!(snapshot.active_sequences(), 0);
4605 assert!(snapshot
4606 .domains()
4607 .iter()
4608 .all(|domain| domain.used().get() == 0));
4609 }
4610
4611 #[test]
4612 fn poisoned_drop_retains_claim_and_is_observable() {
4613 let coordinator = coordinator(8);
4614 let request = request(&coordinator);
4615 let lease = admit_sequence(
4616 &coordinator,
4617 &request,
4618 &demand(&[(1, 2), (2, 1)], &[(1, 2), (2, 1)]),
4619 );
4620 let inner = Arc::clone(&coordinator.inner);
4621 let _ = thread::spawn(move || {
4622 let _guard = inner.state.lock().unwrap();
4623 panic!("poison admission state for conservative-drop test");
4624 })
4625 .join();
4626 drop(lease);
4627 let snapshot = coordinator.snapshot().unwrap();
4628 assert!(snapshot.poisoned());
4629 assert_eq!(snapshot.active_sequences(), 1);
4630 assert_eq!(snapshot.live_sequence_records(), 1);
4631 assert_eq!(snapshot.domains()[0].used().get(), 2);
4632 }
4633}
4634
4635#[cfg(test)]
4636#[path = "admission/model_check.rs"]
4637mod model_check;