1#[cfg(unix)]
31use std::io;
32use std::sync::atomic::{AtomicU8, AtomicU32, AtomicU64, Ordering};
33
34use orbit_core::process::ProcessOwner;
35
36#[path = "inbox/claims.rs"]
37pub(crate) mod claims;
38use claims::{CANCELLED, COMMITTED};
39
40use super::{Error, Result};
41
42pub(crate) const INBOX_MAGIC: u32 = 0x4f54_5849; pub(crate) const INBOX_VERSION: u32 = 13;
82
83#[derive(Clone, Copy, Debug, Eq, PartialEq)]
87pub struct GrowthPolicy {
88 pub min: u16,
89 pub max: u16,
90 pub step: u16,
91}
92
93#[derive(Clone, Copy, Debug, PartialEq, Eq)]
95pub struct Reclaimed {
96 pub lane: usize,
97 pub incarnation: u64,
99 pub pid: u32,
100}
101
102pub const LANE_NAME_MAX: usize = 47;
106
107pub const LANE_ROLE_MAX: usize = 8;
110
111fn accept_role(
113 role: &[u8; LANE_ROLE_MAX],
114 role_len: &AtomicU8,
115 accept: impl FnOnce(&str) -> bool,
116) -> bool {
117 let len = role_len.load(Ordering::Acquire) as usize;
118 if len == 0 || len > LANE_ROLE_MAX {
119 return false;
120 }
121 let mut copy = [0u8; LANE_ROLE_MAX];
122 copy[..len].copy_from_slice(unsafe {
125 std::slice::from_raw_parts(role.as_ptr(), len)
126 });
127 std::str::from_utf8(©[..len]).is_ok_and(accept)
128}
129
130#[repr(C, align(64))]
138pub(crate) struct LaneHeader {
139 pub(crate) reserve: AtomicU64,
142 pub(crate) read: AtomicU64,
144 pub(crate) consumers: AtomicU32,
147 pub(crate) changes: AtomicU32,
149 pub(crate) incarnation: AtomicU64,
153 pub(crate) waiters: AtomicU32,
154 pub(crate) reactor: AtomicU32,
158 pub(crate) epoch: AtomicU64,
159 initialized: AtomicU32,
160 paired_bodies_epoch: AtomicU32,
162
163 pub(crate) service: [u8; LANE_NAME_MAX],
175 pub(crate) service_len: AtomicU8,
176 pub(crate) pid: AtomicU32,
193 pub(crate) role: [u8; LANE_ROLE_MAX],
205 pub(crate) role_len: AtomicU8,
206 _identity_padding: [u8; 3],
207}
208
209const _: () = assert!(size_of::<LaneHeader>() == 128);
211
212#[repr(C, align(64))]
225pub(crate) struct WatcherHeader {
226 pub(crate) changes: AtomicU32,
228 pub(crate) reactor: AtomicU32,
231 pub(crate) pid: AtomicU32,
233 pub(crate) role: [u8; LANE_ROLE_MAX],
236 pub(crate) role_len: AtomicU8,
237 _padding: [u8; 43],
238}
239
240const _: () = assert!(size_of::<WatcherHeader>() == 64);
241
242pub(crate) const INTEREST_WORDS: usize = 7;
244#[cfg_attr(not(feature = "events"), allow(dead_code))]
246pub(crate) const INTEREST_BITS: usize = INTEREST_WORDS * 64;
247
248#[repr(C, align(64))]
256pub(crate) struct InterestEntry {
257 pub(crate) seq: AtomicU32,
258 _padding: [u8; 4],
259 pub(crate) bits: [AtomicU64; INTEREST_WORDS],
260}
261
262const _: () = assert!(size_of::<InterestEntry>() == 64);
263
264#[repr(C, align(64))]
267pub(crate) struct SlotHeader {
268 pub(crate) seq: AtomicU64,
269 pub(crate) claim: AtomicU64,
270 pub(crate) work_epoch: AtomicU64,
271 pub(crate) work_claim: AtomicU64,
272 pub(crate) len: AtomicU32,
273 pub(crate) work_slot: AtomicU32,
274 _padding: [u8; 24],
275}
276
277const _: () = assert!(size_of::<SlotHeader>() == 64);
278
279#[derive(Clone, Copy, Debug, Eq, PartialEq)]
281pub enum Admission {
282 Accepted,
284 Full,
286 NoConsumer,
288}
289
290#[derive(Clone, Copy, Debug, Eq, PartialEq)]
293pub struct InboxGeometry {
294 pub capacity: usize,
295 pub payload_capacity: usize,
296}
297
298impl InboxGeometry {
299 pub const fn new(
300 capacity: usize,
301 payload_capacity: usize,
302 ) -> Self {
303 Self { capacity, payload_capacity }
304 }
305
306 pub const fn lane_bytes(&self) -> usize {
309 size_of::<LaneHeader>() + self.capacity * self.slot_bytes()
310 }
311
312 const fn slot_bytes(&self) -> usize {
313 (size_of::<SlotHeader>() + self.payload_capacity + 63) & !63
316 }
317
318 pub const fn segment_bytes(
323 &self,
324 fleet_capacity: usize,
325 ) -> usize {
326 size_of::<TableHeader>()
327 + fleet_capacity
328 * (self.lane_bytes()
329 + size_of::<WatcherHeader>()
330 + 2 * size_of::<InterestEntry>())
331 + claims::registry_bytes(fleet_capacity, self.capacity)
332 + crate::admission::bytes(fleet_capacity, self.capacity)
333 }
334
335 pub(crate) fn validate(&self) -> Result<()> {
336 if self.capacity == 0 || !self.capacity.is_power_of_two() {
337 return Err(Error::Malformed(
338 "inbox capacity must be a power of two",
339 ));
340 }
341 if self.payload_capacity == 0 {
342 return Err(Error::Malformed(
343 "inbox payload capacity must not be zero",
344 ));
345 }
346 if self.capacity > u32::MAX as usize
347 || self.payload_capacity > u32::MAX as usize
348 {
349 return Err(Error::Malformed(
350 "inbox geometry exceeds its wire fields",
351 ));
352 }
353 Ok(())
354 }
355}
356
357const _: () = assert!(size_of::<TableHeader>() == 128);
359
360#[repr(C, align(64))]
362pub(crate) struct TableHeader {
363 pub(crate) magic: AtomicU32,
364 pub(crate) version: AtomicU32,
365 pub(crate) fleet_capacity: AtomicU32,
366 pub(crate) capacity: AtomicU32,
367 pub(crate) payload_capacity: AtomicU32,
368 pub(crate) directory: AtomicU64,
377 pub(crate) policy: AtomicU32,
381 pub(crate) pressure: AtomicU32,
385 pub(crate) growth_min: AtomicU32,
388 pub(crate) growth_max: AtomicU32,
389 pub(crate) growth_step: AtomicU32,
390 _padding: [u8; 16],
391}
392
393pub struct Inbox {
399 pub(crate) base: *mut u8,
400 pub(crate) lanes: usize,
409 pub(crate) geometry: InboxGeometry,
410 producer: Option<claims::Producer>,
412 #[cfg(unix)]
415 bell: Option<crate::bell::Bell>,
416}
417
418unsafe impl Send for Inbox {}
420unsafe impl Sync for Inbox {}
421
422impl Inbox {
423 pub unsafe fn initialize(
442 base: *mut u8,
443 fleet_capacity: usize,
444 geometry: InboxGeometry,
445 ) -> Result<Self> {
446 geometry.validate()?;
447 if fleet_capacity == 0 || fleet_capacity > u16::MAX as usize {
448 return Err(Error::Malformed(
449 "inbox fleet capacity must fit a nonzero u16",
450 ));
451 }
452 let inbox = Self {
453 base,
454 lanes: fleet_capacity,
455 geometry,
456 producer: None,
457 #[cfg(unix)]
458 bell: None,
459 };
460
461 let header = inbox.table_header();
462 header.fleet_capacity.store(fleet_capacity as u32, Ordering::Relaxed);
463 header.capacity.store(geometry.capacity as u32, Ordering::Relaxed);
464 header
465 .payload_capacity
466 .store(geometry.payload_capacity as u32, Ordering::Relaxed);
467 header.version.store(INBOX_VERSION, Ordering::Relaxed);
468 header.magic.store(INBOX_MAGIC, Ordering::Release);
470 Ok(inbox)
471 }
472
473 pub(crate) unsafe fn persisted_shape(
482 base: *const u8
483 ) -> Result<(usize, InboxGeometry)> {
484 let header = unsafe { &*base.cast::<TableHeader>() };
485 if header.magic.load(Ordering::Acquire) != INBOX_MAGIC {
486 return Err(Error::Malformed(
487 "inbox segment is not an inbox table",
488 ));
489 }
490 if header.version.load(Ordering::Relaxed) != INBOX_VERSION {
491 return Err(Error::Malformed(
492 "inbox segment was written by another layout version",
493 ));
494 }
495 let geometry = InboxGeometry::new(
496 header.capacity.load(Ordering::Relaxed) as usize,
497 header.payload_capacity.load(Ordering::Relaxed) as usize,
498 );
499 geometry.validate()?;
500 Ok((header.fleet_capacity.load(Ordering::Relaxed) as usize, geometry))
501 }
502
503 pub unsafe fn attach(
510 base: *mut u8,
511 fleet_capacity: usize,
512 geometry: InboxGeometry,
513 ) -> Result<Self> {
514 geometry.validate()?;
515 if fleet_capacity == 0 || fleet_capacity > u16::MAX as usize {
516 return Err(Error::Malformed(
517 "inbox fleet capacity must fit a nonzero u16",
518 ));
519 }
520 let inbox = Self {
521 base,
522 lanes: fleet_capacity,
523 geometry,
524 producer: None,
525 #[cfg(unix)]
526 bell: None,
527 };
528 let header = inbox.table_header();
529 if header.magic.load(Ordering::Acquire) != INBOX_MAGIC {
530 return Err(Error::Malformed(
531 "inbox segment is not an inbox table",
532 ));
533 }
534 if header.version.load(Ordering::Relaxed) != INBOX_VERSION {
535 return Err(Error::Malformed(
536 "inbox segment was written by another layout version",
537 ));
538 }
539 if header.fleet_capacity.load(Ordering::Relaxed) as usize
540 != fleet_capacity
541 || header.capacity.load(Ordering::Relaxed) as usize
542 != geometry.capacity
543 || header.payload_capacity.load(Ordering::Relaxed) as usize
544 != geometry.payload_capacity
545 {
546 return Err(Error::Malformed(
547 "inbox segment geometry differs from this build",
548 ));
549 }
550 Ok(inbox)
551 }
552
553 pub fn geometry(&self) -> InboxGeometry {
554 self.geometry
555 }
556
557 pub fn lanes(&self) -> usize {
558 self.lanes
559 }
560
561 pub fn lane_epoch(
563 &self,
564 lane: usize,
565 ) -> Result<u64> {
566 self.check_lane(lane)?;
567 Ok(self.lane_header(lane).epoch.load(Ordering::Acquire))
568 }
569
570 pub fn advertise_paired_bodies(
574 &self,
575 lane: usize,
576 ) -> Result<()> {
577 let epoch = u32::try_from(self.lane_epoch(lane)?).map_err(|_| {
578 Error::Malformed("paired body advertisement epoch exhausted")
579 })?;
580 let previous = self
581 .lane_header(lane)
582 .paired_bodies_epoch
583 .swap(epoch, Ordering::AcqRel);
584 if previous != epoch {
585 self.table_header().directory.fetch_add(1, Ordering::AcqRel);
586 self.announce_directory();
587 }
588 Ok(())
589 }
590
591 pub fn paired_bodies_epoch(
593 &self,
594 lane: usize,
595 ) -> Option<u64> {
596 let epoch = self.lane_epoch(lane).ok()?;
597 let stamp =
598 self.lane_header(lane).paired_bodies_epoch.load(Ordering::Acquire);
599 (stamp != 0 && u64::from(stamp) == epoch).then_some(epoch)
600 }
601
602 pub fn claim_lane(
607 &self,
608 lane_index: usize,
609 service: &str,
610 role: &str,
611 incarnation: u64,
612 ) -> Result<()> {
613 self.claim_lane_with_capacity(
614 lane_index,
615 service,
616 role,
617 incarnation,
618 None,
619 )
620 }
621
622 pub fn claim_lane_with_capacity(
624 &self,
625 lane_index: usize,
626 service: &str,
627 role: &str,
628 incarnation: u64,
629 capacity: Option<usize>,
630 ) -> Result<()> {
631 self.check_lane(lane_index)?;
632 check_identity(service, role)?;
633 self.validate_work_capacity(capacity)?;
634 let pid = ProcessOwner::current()?.pid();
635 self.reset_lane(lane_index)?;
636 self.configure_capacity(lane_index, capacity)?;
637 let lane = self.lane_header(lane_index);
638 lane.consumers.store(1, Ordering::Release);
639 self.publish_identity(lane, service, role, incarnation, pid);
640 self.wake(lane_index, lane);
641 Ok(())
642 }
643
644 pub fn join<H>(
676 &self,
677 service: &str,
678 role: &str,
679 incarnation: u64,
680 hold: impl FnMut(usize) -> Result<Option<H>>,
681 ) -> Result<(usize, H)> {
682 self.join_with_capacity(service, role, incarnation, None, hold)
683 }
684
685 pub fn join_with_capacity<H>(
686 &self,
687 service: &str,
688 role: &str,
689 incarnation: u64,
690 capacity: Option<usize>,
691 mut hold: impl FnMut(usize) -> Result<Option<H>>,
692 ) -> Result<(usize, H)> {
693 check_identity(service, role)?;
694 self.validate_work_capacity(capacity)?;
695 self.refuse_other_role(service, role, None)?;
696 let pid = ProcessOwner::current()?.pid();
697 for lane_index in 0..self.lanes {
698 let lane = self.lane_header(lane_index);
699 if lane.consumers.load(Ordering::Acquire) != 0 {
700 continue;
701 }
702 let Some(held) = hold(lane_index)? else {
703 continue;
704 };
705 if lane
706 .consumers
707 .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire)
708 .is_err()
709 {
710 continue;
711 }
712 self.reset_lane(lane_index)?;
713 self.configure_capacity(lane_index, capacity)?;
714 self.publish_identity(lane, service, role, incarnation, pid);
715 if let Err(conflict) =
716 self.refuse_other_role(service, role, Some(lane_index))
717 {
718 let _ = self.release_lane(lane_index);
719 return Err(conflict);
720 }
721 self.wake(lane_index, lane);
722 return Ok((lane_index, held));
723 }
724 Err(Error::NoLane)
725 }
726
727 pub fn reclaim(
733 &self,
734 lane_index: usize,
735 ) -> Result<Option<Reclaimed>> {
736 self.check_lane(lane_index)?;
737 let lane = self.lane_header(lane_index);
738 if lane.consumers.load(Ordering::Acquire) == 0 {
739 return Ok(None);
740 }
741 let reclaimed = Reclaimed {
742 lane: lane_index,
743 incarnation: lane.incarnation.load(Ordering::Acquire),
744 pid: lane.pid.load(Ordering::Acquire),
745 };
746 self.release_lane(lane_index)?;
747 Ok(Some(reclaimed))
748 }
749
750 fn refuse_other_role(
755 &self,
756 service: &str,
757 role: &str,
758 own: Option<usize>,
759 ) -> Result<()> {
760 for lane_index in self.serving(service) {
761 if Some(lane_index) == own {
762 continue;
763 }
764 let lane = self.lane_header(lane_index);
765 if lane.consumers.load(Ordering::Acquire) == 0 {
766 continue;
767 }
768 let held_by = self.role(lane_index)?.unwrap_or_default();
769 if held_by != role {
770 return Err(Error::RoleConflict {
771 name: service.to_owned(),
772 role: role.to_owned(),
773 held_by: if held_by.is_empty() {
774 "an unnamed role".into()
775 } else {
776 held_by
777 },
778 lane: lane_index,
779 pid: lane.pid.load(Ordering::Acquire),
780 });
781 }
782 }
783 Ok(())
784 }
785
786 pub fn park(
794 &self,
795 lane_index: usize,
796 expected: u32,
797 ) -> Result<()> {
798 self.check_lane(lane_index)?;
799 let lane = self.lane_header(lane_index);
800 lane.waiters.fetch_add(1, Ordering::AcqRel);
804 let waited = orbit_core::sync::wait_word(&lane.changes, expected);
805 lane.waiters.fetch_sub(1, Ordering::AcqRel);
806 waited.map_err(Error::Io)
807 }
808
809 pub fn nudge(
815 &self,
816 lane_index: usize,
817 ) -> Result<()> {
818 self.check_lane(lane_index)?;
819 self.wake(lane_index, self.lane_header(lane_index));
820 Ok(())
821 }
822
823 pub fn service(
825 &self,
826 lane_index: usize,
827 ) -> Result<Option<String>> {
828 self.check_lane(lane_index)?;
829 let lane = self.lane_header(lane_index);
830 let len = lane.service_len.load(Ordering::Acquire) as usize;
831 if len == 0 || len > LANE_NAME_MAX {
832 return Ok(None);
833 }
834 let bytes =
835 unsafe { std::slice::from_raw_parts(lane.service.as_ptr(), len) };
836 Ok(String::from_utf8(bytes.to_vec()).ok())
837 }
838
839 pub fn owner(
841 &self,
842 lane_index: usize,
843 ) -> Result<Option<u32>> {
844 self.check_lane(lane_index)?;
845 if self.consumers(lane_index)? == 0 {
846 return Ok(None);
847 }
848 Ok(match self.lane_header(lane_index).pid.load(Ordering::Acquire) {
849 0 => None,
850 pid => Some(pid),
851 })
852 }
853
854 pub fn last_owner(
857 &self,
858 lane_index: usize,
859 ) -> Result<Option<(u32, u64)>> {
860 self.check_lane(lane_index)?;
861 let lane = self.lane_header(lane_index);
862 let pid = lane.pid.load(Ordering::Acquire);
863 Ok((pid != 0).then(|| (pid, lane.incarnation.load(Ordering::Acquire))))
864 }
865
866 pub fn role_accepted(
871 &self,
872 lane_index: usize,
873 accept: impl FnOnce(&str) -> bool,
874 ) -> bool {
875 if self.check_lane(lane_index).is_err() {
876 return false;
877 }
878 let lane = self.lane_header(lane_index);
879 accept_role(&lane.role, &lane.role_len, accept)
880 }
881
882 pub fn watcher_role_accepted(
885 &self,
886 slot: usize,
887 accept: impl FnOnce(&str) -> bool,
888 ) -> bool {
889 if self.check_lane(slot).is_err() {
890 return false;
891 }
892 let watcher = self.watcher_header(slot);
893 watcher.pid.load(Ordering::Acquire) != 0
894 && accept_role(&watcher.role, &watcher.role_len, accept)
895 }
896
897 pub fn role(
898 &self,
899 lane_index: usize,
900 ) -> Result<Option<String>> {
901 self.check_lane(lane_index)?;
902 let lane = self.lane_header(lane_index);
903 let len = lane.role_len.load(Ordering::Acquire) as usize;
904 if len == 0 || len > LANE_ROLE_MAX {
905 return Ok(None);
906 }
907 let bytes =
908 unsafe { std::slice::from_raw_parts(lane.role.as_ptr(), len) };
909 Ok(String::from_utf8(bytes.to_vec()).ok())
910 }
911
912 fn serves(
919 &self,
920 lane_index: usize,
921 service: &[u8],
922 ) -> bool {
923 let lane = self.lane_header(lane_index);
924 let len = lane.service_len.load(Ordering::Acquire) as usize;
925 if len != service.len() || len > LANE_NAME_MAX {
926 return false;
927 }
928 let name =
929 unsafe { std::slice::from_raw_parts(lane.service.as_ptr(), len) };
930 name == service
931 }
932
933 pub fn targets(
941 &self,
942 service: &str,
943 ) -> Vec<(usize, u64)> {
944 self.ready(&self.serving(service))
945 }
946
947 pub fn directory(&self) -> u64 {
954 self.table_header().directory.load(Ordering::Acquire)
955 }
956
957 pub fn set_growth_policy(
961 &self,
962 policy: Option<GrowthPolicy>,
963 ) {
964 let header = self.table_header();
965 match policy {
966 Some(policy) => {
967 header.growth_min.store(policy.min.into(), Ordering::Relaxed);
968 header.growth_max.store(policy.max.into(), Ordering::Relaxed);
969 header.growth_step.store(policy.step.into(), Ordering::Relaxed);
970 header.policy.store(1, Ordering::Release);
971 }
972 None => header.policy.store(0, Ordering::Release),
973 }
974 }
975
976 pub fn growth_policy(&self) -> Option<GrowthPolicy> {
978 let header = self.table_header();
979 if header.policy.load(Ordering::Acquire) == 0 {
980 return None;
981 }
982 let bound = |word: &AtomicU32| {
983 u16::try_from(word.load(Ordering::Relaxed)).unwrap_or(u16::MAX)
984 };
985 Some(GrowthPolicy {
986 min: bound(&header.growth_min),
987 max: bound(&header.growth_max),
988 step: bound(&header.growth_step),
989 })
990 }
991
992 pub fn report_pressure(&self) -> bool {
998 let header = self.table_header();
999 if header.policy.load(Ordering::Acquire) == 0
1000 || header.pressure.load(Ordering::Acquire) != 0
1001 || header
1002 .pressure
1003 .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire)
1004 .is_err()
1005 {
1006 return false;
1007 }
1008 for slot in 0..self.lanes {
1009 self.wake_watcher(slot);
1010 }
1011 true
1012 }
1013
1014 pub fn pressure_pending(&self) -> bool {
1015 self.table_header().pressure.load(Ordering::Acquire) != 0
1016 }
1017
1018 pub fn take_pressure(&self) -> bool {
1021 self.table_header().pressure.swap(0, Ordering::AcqRel) != 0
1022 }
1023
1024 pub fn serving(
1031 &self,
1032 service: &str,
1033 ) -> Vec<usize> {
1034 let wanted = service.as_bytes();
1035 (0..self.lanes).filter(|lane| self.serves(*lane, wanted)).collect()
1036 }
1037
1038 pub fn ready(
1043 &self,
1044 lanes: &[usize],
1045 ) -> Vec<(usize, u64)> {
1046 let mut found: Vec<(usize, u64)> = lanes
1047 .iter()
1048 .copied()
1049 .filter(|lane| {
1050 matches!(self.admits(*lane), Ok(Admission::Accepted))
1051 })
1052 .filter_map(|lane| self.depth(lane).ok().map(|depth| (lane, depth)))
1053 .collect();
1054 found.sort_by_key(|(_, depth)| *depth);
1055 found
1056 }
1057
1058 pub fn idlest(
1061 &self,
1062 lanes: &[usize],
1063 ) -> Option<(usize, u64)> {
1064 lanes
1065 .iter()
1066 .copied()
1067 .filter(|lane| {
1068 matches!(self.admits(*lane), Ok(Admission::Accepted))
1069 })
1070 .filter_map(|lane| self.depth(lane).ok().map(|depth| (lane, depth)))
1071 .min_by_key(|(_, depth)| *depth)
1072 }
1073
1074 pub fn release_lane(
1078 &self,
1079 lane_index: usize,
1080 ) -> Result<()> {
1081 self.check_lane(lane_index)?;
1082 let lane = self.lane_header(lane_index);
1083 lane.consumers.store(0, Ordering::Release);
1084 lane.service_len.store(0, Ordering::Release);
1085 lane.role_len.store(0, Ordering::Release);
1086 self.clear_interest(lane_index);
1090 std::sync::atomic::fence(Ordering::SeqCst);
1091 self.capacity_changed(lane_index);
1092 self.table_header().directory.fetch_add(1, Ordering::AcqRel);
1093 self.wake(lane_index, lane);
1094 self.announce_directory();
1095 Ok(())
1096 }
1097
1098 pub fn admits(
1103 &self,
1104 lane_index: usize,
1105 ) -> Result<Admission> {
1106 self.check_lane(lane_index)?;
1107 let lane = self.lane_header(lane_index);
1108 if lane.consumers.load(Ordering::Acquire) == 0 {
1109 return Ok(Admission::NoConsumer);
1110 }
1111 let read = lane.read.load(Ordering::Acquire);
1112 let reserve = lane.reserve.load(Ordering::Acquire);
1118 if reserve.wrapping_sub(read) >= self.geometry.capacity as u64 {
1119 return Ok(Admission::Full);
1120 }
1121 Ok(Admission::Accepted)
1122 }
1123
1124 pub fn depth(
1130 &self,
1131 lane_index: usize,
1132 ) -> Result<u64> {
1133 self.check_lane(lane_index)?;
1134 let lane = self.lane_header(lane_index);
1135 let read = lane.read.load(Ordering::Acquire);
1136 let reserve = lane.reserve.load(Ordering::Acquire);
1137 Ok(reserve.wrapping_sub(read))
1138 }
1139
1140 pub fn consumers(
1141 &self,
1142 lane_index: usize,
1143 ) -> Result<u32> {
1144 self.check_lane(lane_index)?;
1145 Ok(self.lane_header(lane_index).consumers.load(Ordering::Acquire))
1146 }
1147
1148 pub fn incarnation(
1149 &self,
1150 lane_index: usize,
1151 ) -> Result<u64> {
1152 self.check_lane(lane_index)?;
1153 Ok(self.lane_header(lane_index).incarnation.load(Ordering::Acquire))
1154 }
1155
1156 pub fn write(
1165 &self,
1166 lane_index: usize,
1167 payload: &[u8],
1168 ) -> Result<u64> {
1169 self.write_vectored(lane_index, &[payload])
1170 }
1171
1172 pub fn write_vectored(
1176 &self,
1177 lane_index: usize,
1178 parts: &[&[u8]],
1179 ) -> Result<u64> {
1180 self.write_frame(lane_index, parts, None, None, false)
1181 }
1182
1183 pub fn write_vectored_at_epoch(
1186 &self,
1187 lane: usize,
1188 epoch: u64,
1189 parts: &[&[u8]],
1190 ) -> Result<u64> {
1191 self.write_frame(lane, parts, None, Some(epoch), false)
1192 }
1193
1194 pub fn write_reply_at_epoch(
1196 &self,
1197 lane: usize,
1198 epoch: u64,
1199 parts: &[&[u8]],
1200 ) -> Result<u64> {
1201 self.write_frame(lane, parts, None, Some(epoch), true)
1202 }
1203
1204 pub(crate) fn write_work(
1205 &self,
1206 ticket: crate::WorkTicket,
1207 parts: &[&[u8]],
1208 ) -> Result<u64> {
1209 self.check_work(ticket)?;
1210 self.write_frame(
1211 usize::from(ticket.lane),
1212 parts,
1213 Some(ticket),
1214 Some(ticket.epoch),
1215 false,
1216 )
1217 }
1218
1219 fn write_frame(
1220 &self,
1221 lane_index: usize,
1222 parts: &[&[u8]],
1223 work: Option<crate::WorkTicket>,
1224 epoch: Option<u64>,
1225 reply: bool,
1226 ) -> Result<u64> {
1227 self.check_lane(lane_index)?;
1228 if reply
1229 && (self.lane_epoch(lane_index)? != epoch.unwrap_or_default()
1230 || self.consumers(lane_index)? == 0)
1231 {
1232 return Err(Error::NoConsumer);
1233 }
1234 let len = parts
1235 .iter()
1236 .try_fold(0usize, |len, part| len.checked_add(part.len()))
1237 .ok_or(Error::FrameTooLarge)?;
1238 if len > self.geometry.payload_capacity {
1239 return Err(Error::FrameTooLarge);
1240 }
1241 let mut reservation = match self.reserve_frame(lane_index) {
1242 Err(Error::InboxContended) if !reply => {
1243 return Err(Error::InboxFull);
1244 }
1245 result => result?,
1246 };
1247 if epoch.is_some_and(|epoch| epoch != reservation.epoch)
1248 || (reply && self.consumers(lane_index)? == 0)
1249 {
1250 return Err(Error::NoConsumer);
1251 }
1252 let position = reservation.position;
1253 let slot = self.slot_header(lane_index, position);
1254 let mut at = self.payload_ptr(lane_index, position);
1255 for part in parts {
1256 unsafe {
1259 std::ptr::copy_nonoverlapping(part.as_ptr(), at, part.len());
1260 at = at.add(part.len());
1261 }
1262 }
1263 #[cfg(test)]
1264 claims::checkpoint("payload");
1265 if let Some(ticket) = work {
1266 slot.work_epoch.store(ticket.epoch, Ordering::Relaxed);
1267 slot.work_slot.store(ticket.slot, Ordering::Relaxed);
1268 slot.work_claim.store(ticket.claim, Ordering::Relaxed);
1269 } else {
1270 slot.work_claim.store(0, Ordering::Relaxed);
1271 }
1272 slot.len.store(len as u32, Ordering::Relaxed);
1273 slot.claim.store(reservation.claim(COMMITTED), Ordering::Release);
1275 #[cfg(test)]
1276 claims::checkpoint("committed");
1277 reservation.committed = true;
1278 self.wake(lane_index, self.lane_header(lane_index));
1279 Ok(position)
1280 }
1281
1282 pub fn reserved_through(
1286 &self,
1287 lane: usize,
1288 ) -> Result<u64> {
1289 self.check_lane(lane)?;
1290 Ok(self.lane_header(lane).reserve.load(Ordering::Acquire))
1291 }
1292
1293 pub fn read_position(
1295 &self,
1296 lane: usize,
1297 ) -> Result<u64> {
1298 self.check_lane(lane)?;
1299 Ok(self.lane_header(lane).read.load(Ordering::Acquire))
1300 }
1301
1302 pub fn read(
1308 &self,
1309 lane_index: usize,
1310 out: &mut Vec<u8>,
1311 ) -> Result<bool> {
1312 match self.read_frame(lane_index, out)? {
1313 None => Ok(false),
1314 Some(None) => Ok(true),
1315 Some(Some(ticket)) => {
1316 self.cancel_work(ticket);
1317 Err(Error::Malformed(
1318 "work frames require an admission-aware receiver",
1319 ))
1320 }
1321 }
1322 }
1323
1324 pub fn read_frame(
1327 &self,
1328 lane_index: usize,
1329 out: &mut Vec<u8>,
1330 ) -> Result<Option<Option<crate::WorkTicket>>> {
1331 self.read_frame_before(lane_index, out, None)
1332 }
1333
1334 pub(crate) fn read_frame_before(
1335 &self,
1336 lane_index: usize,
1337 out: &mut Vec<u8>,
1338 boundary: Option<u64>,
1339 ) -> Result<Option<Option<crate::WorkTicket>>> {
1340 self.check_lane(lane_index)?;
1341 let lane = self.lane_header(lane_index);
1342 for _ in 0..self.geometry.capacity {
1343 let position = lane.read.load(Ordering::Relaxed);
1344 if boundary.is_some_and(|boundary| position >= boundary)
1345 || position >= lane.reserve.load(Ordering::Acquire)
1346 {
1347 return Ok(None);
1348 }
1349 let slot = self.slot_header(lane_index, position);
1350 let claim = slot.claim.load(Ordering::Acquire);
1351 if slot.seq.load(Ordering::Acquire) != position
1352 || !matches!(claim & 3, COMMITTED | CANCELLED)
1353 {
1354 return Ok(None);
1355 }
1356 let intent = self.intent_for_claim(claim)?;
1357 if intent.position != position || intent.lane != lane_index {
1358 return Err(Error::Malformed(
1359 "committed inbox intent has the wrong turn",
1360 ));
1361 }
1362 let deliver = claim & 3 == COMMITTED
1363 && intent.incarnation == lane.epoch.load(Ordering::Acquire);
1364 let work_claim = slot.work_claim.load(Ordering::Relaxed);
1365 let work = (work_claim != 0).then(|| crate::WorkTicket {
1366 lane: lane_index as u16,
1367 epoch: slot.work_epoch.load(Ordering::Relaxed),
1368 slot: slot.work_slot.load(Ordering::Relaxed),
1369 claim: work_claim,
1370 });
1371 if deliver {
1372 let len = slot.len.load(Ordering::Relaxed) as usize;
1373 if len > self.geometry.payload_capacity {
1374 return Err(Error::Malformed(
1375 "committed inbox length is invalid",
1376 ));
1377 }
1378 out.clear();
1379 out.reserve(len);
1380 unsafe {
1381 std::ptr::copy_nonoverlapping(
1382 self.payload_ptr(lane_index, position),
1383 out.spare_capacity_mut().as_mut_ptr().cast::<u8>(),
1384 len,
1385 );
1386 out.set_len(len);
1387 }
1388 }
1389 self.consume_frame(lane_index, position, claim)?;
1390 if deliver {
1391 return Ok(Some(work));
1392 }
1393 }
1394 Ok(None)
1395 }
1396
1397 pub fn wait_word(
1399 &self,
1400 lane_index: usize,
1401 ) -> Result<(&AtomicU32, u32)> {
1402 self.check_lane(lane_index)?;
1403 let lane = self.lane_header(lane_index);
1404 Ok((&lane.changes, lane.changes.load(Ordering::Acquire)))
1405 }
1406
1407 pub(crate) fn retire_lane(
1410 &self,
1411 lane_index: usize,
1412 ) -> Result<()> {
1413 self.check_lane(lane_index)?;
1414 if self.consumers(lane_index)? != 0 {
1415 return Err(Error::Malformed("retiring an advertised inbox lane"));
1416 }
1417 self.reset_lane(lane_index)
1418 }
1419
1420 fn reset_lane(
1421 &self,
1422 lane_index: usize,
1423 ) -> Result<()> {
1424 let lane = self.lane_header(lane_index);
1425 let next_epoch = lane
1428 .epoch
1429 .load(Ordering::Acquire)
1430 .checked_add(1)
1431 .ok_or(Error::Malformed("inbox lane generation exhausted"))?;
1432 if lane.initialized.load(Ordering::Acquire) == 0 {
1433 for position in 0..self.geometry.capacity {
1434 let slot = self.slot_header(lane_index, position as u64);
1435 slot.seq.store(position as u64, Ordering::Release);
1436 }
1437 lane.initialized.store(1, Ordering::Release);
1438 } else {
1439 for _ in 0..self.geometry.capacity {
1440 let position = lane.read.load(Ordering::Relaxed);
1441 if position >= lane.reserve.load(Ordering::Acquire) {
1442 break;
1443 }
1444 if self
1445 .slot_header(lane_index, position)
1446 .seq
1447 .load(Ordering::Acquire)
1448 > position
1449 {
1450 let claim = self
1453 .slot_header(lane_index, position)
1454 .claim
1455 .load(Ordering::Acquire);
1456 if matches!(claim & 3, COMMITTED | CANCELLED)
1457 && self.intent_for_claim(claim)?.position == position
1458 {
1459 self.consume_frame(lane_index, position, claim)?;
1460 } else {
1461 lane.read.store(position + 1, Ordering::Release);
1462 }
1463 continue;
1464 }
1465 let claim = self
1466 .slot_header(lane_index, position)
1467 .claim
1468 .load(Ordering::Acquire);
1469 if !matches!(claim & 3, COMMITTED | CANCELLED) {
1470 break;
1471 }
1472 self.consume_frame(lane_index, position, claim)?;
1473 }
1474 }
1475 lane.service_len.store(0, Ordering::Relaxed);
1479 lane.epoch.store(next_epoch, Ordering::Release);
1480 self.advertise_inbox_space(lane_index, next_epoch);
1481 std::sync::atomic::fence(Ordering::SeqCst);
1482 self.inbox_space_changed(lane_index);
1483 lane.role_len.store(0, Ordering::Relaxed);
1484 lane.waiters.store(0, Ordering::Relaxed);
1485 lane.reactor.store(0, Ordering::Relaxed);
1486 self.clear_interest(lane_index);
1487 self.recover_work(lane_index);
1488 self.configure_capacity(lane_index, None)?;
1489 Ok(())
1490 }
1491
1492 fn publish_identity(
1495 &self,
1496 lane: &LaneHeader,
1497 service: &str,
1498 role: &str,
1499 incarnation: u64,
1500 pid: u32,
1501 ) {
1502 unsafe {
1503 std::ptr::copy_nonoverlapping(
1504 service.as_ptr(),
1505 lane.service.as_ptr().cast_mut(),
1506 service.len(),
1507 );
1508 std::ptr::copy_nonoverlapping(
1509 role.as_ptr(),
1510 lane.role.as_ptr().cast_mut(),
1511 role.len(),
1512 );
1513 }
1514 lane.role_len.store(role.len() as u8, Ordering::Release);
1517 lane.service_len.store(service.len() as u8, Ordering::Release);
1518 lane.incarnation.store(incarnation, Ordering::Release);
1519 lane.pid.store(pid, Ordering::Release);
1520 self.table_header().directory.fetch_add(1, Ordering::AcqRel);
1523 self.announce_directory();
1524 }
1525
1526 fn announce_directory(&self) {
1533 for lane_index in 0..self.lanes {
1534 let lane = self.lane_header(lane_index);
1535 if lane.consumers.load(Ordering::Acquire) != 0 {
1536 self.wake(lane_index, lane);
1537 }
1538 }
1539 for slot in 0..self.lanes {
1540 self.wake_watcher(slot);
1541 }
1542 }
1543
1544 fn wake_watcher(
1547 &self,
1548 slot: usize,
1549 ) {
1550 let watcher = self.watcher_header(slot);
1551 watcher.changes.fetch_add(1, Ordering::SeqCst);
1552 #[cfg(unix)]
1553 if let Some(bell) = &self.bell
1554 && let Err(error) = ring_armed(&watcher.reactor, || {
1555 bell.ring(crate::bell::Doorbell::Watcher(slot))
1556 })
1557 {
1558 tracing::warn!(slot, %error, "Link watcher wake failed");
1559 }
1560 }
1561
1562 pub(crate) fn wake(
1568 &self,
1569 lane_index: usize,
1570 lane: &LaneHeader,
1571 ) {
1572 lane.changes.fetch_add(1, Ordering::SeqCst);
1573 if lane.waiters.load(Ordering::SeqCst) > 0 {
1574 let _ = orbit_core::sync::wake_word(&lane.changes);
1575 }
1576 #[cfg(unix)]
1577 if let Some(bell) = &self.bell
1578 && let Err(error) = ring_armed(&lane.reactor, || {
1579 bell.ring_lane(lane_index, lane.epoch.load(Ordering::Acquire))
1580 })
1581 {
1582 tracing::warn!(lane = lane_index, %error, "Link lane wake failed");
1583 }
1584 }
1585
1586 #[cfg(unix)]
1589 pub(crate) fn ring_adopter(&self) {
1590 if let Some(bell) = &self.bell {
1591 let _ = bell.ring(crate::bell::Doorbell::Adopter);
1592 }
1593 }
1594
1595 #[cfg(unix)]
1596 pub(crate) fn install_bell_with_group(
1597 &mut self,
1598 segment: &str,
1599 group: Option<orbit_core::shm::GroupAccess>,
1600 ) -> Result<()> {
1601 self.bell = Some(
1602 crate::bell::Bell::with_group(segment, group, self.lanes)
1603 .map_err(Error::Io)?,
1604 );
1605 Ok(())
1606 }
1607
1608 #[cfg(all(unix, feature = "tokio"))]
1611 pub(crate) fn bind_producer(&self) -> Result<tokio::net::UnixDatagram> {
1612 let index = self
1613 .producer_index()
1614 .ok_or(Error::Malformed("this inbox mapping has no producer"))?;
1615 let bell = self
1616 .bell
1617 .as_ref()
1618 .ok_or(Error::Malformed("the inbox has no reactor bell"))?;
1619 let socket = bell
1620 .bind(crate::bell::Doorbell::Producer(index))
1621 .map_err(Error::Io)?;
1622 tokio::net::UnixDatagram::from_std(socket).map_err(Error::Io)
1623 }
1624
1625 #[cfg(all(unix, feature = "tokio"))]
1626 pub(crate) fn unbind_producer(&self) {
1627 if let (Some(index), Some(bell)) = (self.producer_index(), &self.bell) {
1628 bell.unbind(crate::bell::Doorbell::Producer(index));
1629 }
1630 }
1631
1632 #[cfg(all(unix, feature = "tokio"))]
1633 pub(crate) fn bind_receiver(
1634 &self,
1635 lane_index: usize,
1636 ) -> Result<tokio::net::UnixDatagram> {
1637 self.check_lane(lane_index)?;
1638 let bell = self
1639 .bell
1640 .as_ref()
1641 .ok_or(Error::Malformed("the inbox has no reactor bell"))?;
1642 let socket = bell
1643 .bind(crate::bell::Doorbell::Lane(lane_index))
1644 .map_err(Error::Io)?;
1645 tokio::net::UnixDatagram::from_std(socket).map_err(Error::Io)
1646 }
1647
1648 #[cfg(all(unix, feature = "tokio"))]
1649 pub(crate) fn unbind_receiver(
1650 &self,
1651 lane_index: usize,
1652 ) {
1653 self.disarm_receiver(lane_index);
1654 if let Some(bell) = &self.bell {
1655 bell.unbind(crate::bell::Doorbell::Lane(lane_index));
1656 }
1657 }
1658
1659 #[cfg(all(unix, feature = "tokio"))]
1660 pub(crate) fn receiver_generation(
1661 &self,
1662 lane_index: usize,
1663 ) -> Result<u32> {
1664 self.check_lane(lane_index)?;
1665 Ok(self.lane_header(lane_index).changes.load(Ordering::SeqCst))
1666 }
1667
1668 #[cfg(all(unix, feature = "tokio"))]
1669 pub(crate) fn arm_receiver(
1670 &self,
1671 lane_index: usize,
1672 ) -> Result<()> {
1673 self.check_lane(lane_index)?;
1674 self.lane_header(lane_index).reactor.store(1, Ordering::SeqCst);
1675 Ok(())
1676 }
1677
1678 #[cfg(all(unix, feature = "tokio"))]
1681 pub(crate) fn take_receiver_arm(
1682 &self,
1683 lane_index: usize,
1684 ) -> bool {
1685 lane_index < self.lanes
1686 && self.lane_header(lane_index).reactor.swap(0, Ordering::SeqCst)
1687 != 0
1688 }
1689
1690 #[cfg(all(unix, feature = "tokio"))]
1691 pub(crate) fn disarm_receiver(
1692 &self,
1693 lane_index: usize,
1694 ) {
1695 if lane_index < self.lanes {
1696 self.lane_header(lane_index).reactor.store(0, Ordering::SeqCst);
1697 }
1698 }
1699
1700 #[cfg(all(unix, feature = "tokio"))]
1703 pub(crate) fn bind_watcher(
1704 &self,
1705 slot: usize,
1706 role: &str,
1707 ) -> Result<tokio::net::UnixDatagram> {
1708 self.check_lane(slot)?;
1709 if role.is_empty() || role.len() > LANE_ROLE_MAX || !role.is_ascii() {
1710 return Err(Error::Malformed("role tag does not fit a watcher"));
1711 }
1712 let bell = self
1713 .bell
1714 .as_ref()
1715 .ok_or(Error::Malformed("the inbox has no reactor bell"))?;
1716 let watcher = self.watcher_header(slot);
1717 let pid = ProcessOwner::current()?.pid();
1718 watcher.reactor.store(0, Ordering::SeqCst);
1719 self.clear_interest(self.lanes + slot);
1720 let socket = bell
1721 .bind(crate::bell::Doorbell::Watcher(slot))
1722 .map_err(Error::Io)?;
1723 unsafe {
1724 std::ptr::copy_nonoverlapping(
1725 role.as_ptr(),
1726 watcher.role.as_ptr().cast_mut(),
1727 role.len(),
1728 );
1729 }
1730 watcher.role_len.store(role.len() as u8, Ordering::Release);
1731 watcher.pid.store(pid, Ordering::Release);
1732 tokio::net::UnixDatagram::from_std(socket).map_err(Error::Io)
1733 }
1734
1735 #[cfg(all(unix, feature = "tokio"))]
1736 pub(crate) fn unbind_watcher(
1737 &self,
1738 slot: usize,
1739 ) {
1740 if slot >= self.lanes {
1741 return;
1742 }
1743 let watcher = self.watcher_header(slot);
1744 watcher.reactor.store(0, Ordering::SeqCst);
1745 watcher.pid.store(0, Ordering::Release);
1746 watcher.role_len.store(0, Ordering::Release);
1747 self.clear_interest(self.lanes + slot);
1748 if let Some(bell) = &self.bell {
1749 bell.unbind(crate::bell::Doorbell::Watcher(slot));
1750 }
1751 }
1752
1753 #[cfg(all(unix, feature = "tokio"))]
1754 pub(crate) fn watcher_generation(
1755 &self,
1756 slot: usize,
1757 ) -> Result<u32> {
1758 self.check_lane(slot)?;
1759 Ok(self.watcher_header(slot).changes.load(Ordering::SeqCst))
1760 }
1761
1762 #[cfg(all(unix, feature = "tokio"))]
1763 pub(crate) fn arm_watcher(
1764 &self,
1765 slot: usize,
1766 ) -> Result<()> {
1767 self.check_lane(slot)?;
1768 self.watcher_header(slot).reactor.store(1, Ordering::SeqCst);
1769 Ok(())
1770 }
1771
1772 #[cfg(all(unix, feature = "tokio"))]
1773 pub(crate) fn disarm_watcher(
1774 &self,
1775 slot: usize,
1776 ) {
1777 if slot < self.lanes {
1778 self.watcher_header(slot).reactor.store(0, Ordering::SeqCst);
1779 }
1780 }
1781
1782 pub fn watcher(
1784 &self,
1785 slot: usize,
1786 ) -> Result<Option<u32>> {
1787 self.check_lane(slot)?;
1788 Ok(match self.watcher_header(slot).pid.load(Ordering::Acquire) {
1789 0 => None,
1790 pid => Some(pid),
1791 })
1792 }
1793
1794 pub fn watcher_role(
1796 &self,
1797 slot: usize,
1798 ) -> Result<Option<String>> {
1799 self.check_lane(slot)?;
1800 let watcher = self.watcher_header(slot);
1801 if watcher.pid.load(Ordering::Acquire) == 0 {
1802 return Ok(None);
1803 }
1804 let len = watcher.role_len.load(Ordering::Acquire) as usize;
1805 if len == 0 || len > LANE_ROLE_MAX {
1806 return Ok(None);
1807 }
1808 let bytes =
1809 unsafe { std::slice::from_raw_parts(watcher.role.as_ptr(), len) };
1810 Ok(String::from_utf8(bytes.to_vec()).ok())
1811 }
1812
1813 pub(crate) fn write_interest(
1816 &self,
1817 entry: usize,
1818 bits: &[u64; INTEREST_WORDS],
1819 ) {
1820 let interest = self.interest_entry(entry);
1821 let writing = interest.seq.load(Ordering::Relaxed) | 1;
1825 interest.seq.store(writing, Ordering::Release);
1826 for (word, value) in interest.bits.iter().zip(bits) {
1827 word.store(*value, Ordering::Release);
1828 }
1829 interest.seq.store(writing.wrapping_add(1), Ordering::Release);
1830 }
1831
1832 #[cfg_attr(not(feature = "events"), allow(dead_code))]
1840 pub(crate) fn read_interest(
1841 &self,
1842 entry: usize,
1843 ) -> [u64; INTEREST_WORDS] {
1844 let interest = self.interest_entry(entry);
1845 let before = interest.seq.load(Ordering::Acquire);
1846 if before.is_multiple_of(2) {
1847 let mut bits = [0u64; INTEREST_WORDS];
1848 for (value, word) in bits.iter_mut().zip(&interest.bits) {
1849 *value = word.load(Ordering::Acquire);
1850 }
1851 if interest.seq.load(Ordering::Acquire) == before {
1852 return bits;
1853 }
1854 }
1855 [u64::MAX; INTEREST_WORDS]
1856 }
1857
1858 fn clear_interest(
1859 &self,
1860 entry: usize,
1861 ) {
1862 self.write_interest(entry, &[0; INTEREST_WORDS]);
1863 }
1864
1865 fn interest_entry(
1866 &self,
1867 entry: usize,
1868 ) -> &InterestEntry {
1869 unsafe {
1870 &*self
1871 .base
1872 .add(
1873 size_of::<TableHeader>()
1874 + self.lanes
1875 * (self.geometry.lane_bytes()
1876 + size_of::<WatcherHeader>())
1877 + entry * size_of::<InterestEntry>(),
1878 )
1879 .cast::<InterestEntry>()
1880 }
1881 }
1882
1883 fn watcher_header(
1884 &self,
1885 slot: usize,
1886 ) -> &WatcherHeader {
1887 unsafe {
1888 &*self
1889 .base
1890 .add(
1891 size_of::<TableHeader>()
1892 + self.lanes * self.geometry.lane_bytes()
1893 + slot * size_of::<WatcherHeader>(),
1894 )
1895 .cast::<WatcherHeader>()
1896 }
1897 }
1898
1899 pub(crate) fn check_lane(
1900 &self,
1901 lane_index: usize,
1902 ) -> Result<()> {
1903 if lane_index >= self.lanes {
1904 return Err(Error::Malformed("lane is outside the table"));
1905 }
1906 Ok(())
1907 }
1908
1909 fn table_header(&self) -> &TableHeader {
1910 unsafe { &*self.base.cast::<TableHeader>() }
1911 }
1912
1913 fn lane_base(
1914 &self,
1915 lane_index: usize,
1916 ) -> *mut u8 {
1917 unsafe {
1918 self.base.add(
1919 size_of::<TableHeader>()
1920 + lane_index * self.geometry.lane_bytes(),
1921 )
1922 }
1923 }
1924
1925 pub(crate) fn lane_header(
1926 &self,
1927 lane_index: usize,
1928 ) -> &LaneHeader {
1929 unsafe { &*self.lane_base(lane_index).cast::<LaneHeader>() }
1930 }
1931
1932 fn slot_base(
1933 &self,
1934 lane_index: usize,
1935 position: u64,
1936 ) -> *mut u8 {
1937 let index = (position as usize) & (self.geometry.capacity - 1);
1938 let stride = self.geometry.slot_bytes();
1939 unsafe {
1940 self.lane_base(lane_index)
1941 .add(size_of::<LaneHeader>() + index * stride)
1942 }
1943 }
1944
1945 fn slot_header(
1946 &self,
1947 lane_index: usize,
1948 position: u64,
1949 ) -> &SlotHeader {
1950 unsafe { &*self.slot_base(lane_index, position).cast::<SlotHeader>() }
1951 }
1952
1953 fn payload_ptr(
1954 &self,
1955 lane_index: usize,
1956 position: u64,
1957 ) -> *mut u8 {
1958 unsafe {
1959 self.slot_base(lane_index, position).add(size_of::<SlotHeader>())
1960 }
1961 }
1962}
1963
1964fn check_identity(
1967 service: &str,
1968 role: &str,
1969) -> Result<()> {
1970 if service.is_empty() || service.len() > LANE_NAME_MAX {
1971 return Err(Error::Malformed("service name does not fit a lane"));
1972 }
1973 if role.is_empty() || role.len() > LANE_ROLE_MAX || !role.is_ascii() {
1974 return Err(Error::Malformed("role tag does not fit a lane"));
1975 }
1976 Ok(())
1977}
1978
1979#[cfg(unix)]
1982fn ring_armed(
1983 reactor: &AtomicU32,
1984 ring: impl FnOnce() -> io::Result<()>,
1985) -> io::Result<()> {
1986 if reactor.swap(0, Ordering::SeqCst) == 0 {
1987 return Ok(());
1988 }
1989 if let Err(error) = ring() {
1990 reactor.fetch_or(1, Ordering::SeqCst);
1991 return Err(error);
1992 }
1993 Ok(())
1994}
1995
1996#[cfg(all(test, unix, feature = "tokio"))]
1997#[path = "../tests/unit/bell.rs"]
1998mod bell_tests;