1use std::{
4 collections::{BTreeMap, BTreeSet, VecDeque},
5 sync::{
6 Arc, Mutex,
7 atomic::{AtomicUsize, Ordering},
8 },
9 time::Duration,
10};
11
12use arachne_node::{
13 AdmissionReport, ConnectionBudget, ControlClient, NetworkProfile, Node, Permissions,
14 RelayOptions, Topic,
15};
16use serde::{Deserialize, Serialize};
17use serde_json::{Value, json};
18use sha2::{Digest, Sha256};
19use tokio::{
20 runtime::Runtime,
21 sync::{mpsc, watch},
22};
23
24mod client;
25mod committed_view;
26mod interest;
27mod membership;
28
29#[doc(hidden)]
34pub mod harness {
35 pub use crate::membership::StateBasis;
36 pub use crate::membership::wire::{Query, decode_reply, encode_query};
37 pub use crate::presence::harness_presence_packet;
38}
39mod admission_waiters;
40mod persistence;
41pub(crate) mod presence;
42mod protected;
43mod resources;
44mod work_signal;
45mod workspace_activity;
46pub use client::{
47 AdmissionAuthorization, AdmissionReply, Client, ClientConfig, ConnectivityReport,
48 ConnectionCapacityMetrics, ControlTimingMetrics, DeliveryFailure, DeliveryReport,
49 DurationSummary, EndpointInfo, Error, ErrorKind, InterestObservation, InvitationDetails,
50 InvitationInfo, JoinAdmissionStep, JoinRequest, MemberInfo, MemberKind,
51 MembershipGossipMetrics, MemberRoster, Network, PeerPolicy, PeerRoute, Presence, Publication,
52 PublicationCandidate, RecoveredPublication, RecoveryAdoption, RecoveryCandidate,
53 RecoveryRangeReady, RecoveryRangeRequest, RecoveryRangeStatus, RecoveryStage,
54 Result as ClientResult, RouteHint, RouteKind, WorkspaceCandidate, WorkspaceInfo,
55 WorkspaceMetrics, WorkspaceState,
56};
57pub use persistence::{enable_record_storage, restore_record_storage, save_candidate};
58pub use workspace_activity::{Activity as WorkspaceActivity, Phase as WorkspacePhase};
59
60enum WorkspaceTransition {
61 Recovery(
62 Vec<(
63 arachne_routing::PublicationContext,
64 arachne_security::ApplicationMessage,
65 )>,
66 ),
67 RoutedPublication(
68 arachne_routing::PublicationContext,
69 arachne_node::DeliveryClass,
70 Vec<u8>,
71 Vec<[u8; 32]>,
72 Vec<[u8; 32]>,
73 ),
74 RoutedReception(
75 arachne_routing::PublicationContext,
76 arachne_security::ApplicationMessage,
77 Vec<[u8; 32]>,
78 ),
79 Inbox,
80 InboxRejected,
81 InboxRecovery {
82 count: usize,
83 },
84 DirectMiss {
85 missing: u64,
86 },
87 CurrentView {
88 cut: u64,
89 pending: usize,
90 stale: usize,
91 },
92 Admission,
93 Management(arachne_security::ManagementAction, Vec<u8>),
94 WorkspaceName,
95 Invitation(
96 Box<arachne_security::Invitation>,
97 Vec<u8>,
98 arachne_security::ManagementAction,
99 Vec<u8>,
100 ),
101 Join,
102 Publication(Vec<u8>),
103 Reception(arachne_security::ApplicationMessage),
104}
105
106#[derive(Clone, Serialize, Deserialize)]
110struct JoinLifecycle {
111 peers: Vec<[u8; 32]>,
112 selected: Option<[u8; 32]>,
113 #[serde(default)]
114 cursor: usize,
115}
116
117impl JoinLifecycle {
118 fn new(peers: Vec<[u8; 32]>) -> Result<Self, String> {
119 if peers.is_empty()
120 || peers.len() > 3
121 || peers.iter().any(|peer| peer.iter().all(|byte| *byte == 0))
122 || peers.windows(2).any(|pair| pair[0] == pair[1])
123 {
124 return Err("join lifecycle requires one to three distinct Iroh peers".into());
125 }
126 Ok(Self {
127 peers,
128 selected: None,
129 cursor: 0,
130 })
131 }
132
133 fn validate(&self) -> Result<(), String> {
134 if self.peers.is_empty()
135 || self.peers.len() > 3
136 || self.cursor > self.peers.len()
137 || self
138 .peers
139 .iter()
140 .any(|peer| peer.iter().all(|byte| *byte == 0))
141 || self.peers.windows(2).any(|pair| pair[0] == pair[1])
142 || self
143 .selected
144 .is_some_and(|peer| !self.peers.contains(&peer))
145 {
146 return Err("invalid persisted join lifecycle".into());
147 }
148 Ok(())
149 }
150
151 fn advance(&mut self) {
152 self.selected = None;
153 self.cursor = (self.cursor + 1) % self.peers.len();
154 }
155}
156
157struct StagedWorkspace {
158 publisher: Option<arachne_delivery::PublisherLog>,
159 received: Option<arachne_delivery::receive::ReceiveJournal>,
160 inbox: Option<arachne_delivery::inbox::ObjectInbox>,
161 transition: WorkspaceTransition,
162 workspace: arachne_security::Workspace,
163 snapshot: Vec<u8>,
164}
165
166struct QueuedAdmission {
167 checkpoint: Option<Vec<u8>>,
168 display_name: Option<String>,
169 approval_automatic: Option<bool>,
170 validated: arachne_security::ValidatedAdmission,
171}
172
173struct PendingAdmissionApproval {
174 attempt: arachne_security::AdmissionAttempt,
175 queued: QueuedAdmission,
176 delivered: bool,
177 acknowledged: bool,
178}
179
180struct PendingControl<Q> {
181 query: Q,
182 peer: [u8; 32],
183 task: tokio::task::JoinHandle<Result<Vec<u8>, arachne_node::Error>>,
184}
185impl<Q> Drop for PendingControl<Q> {
186 fn drop(&mut self) {
187 self.task.abort();
188 }
189}
190
191enum JoinAttemptOutcome {
192 NotSent,
193 Waiting,
194 Reply {
195 value: Value,
196 history_prefix: Vec<Value>,
197 },
198 Failed(String),
199}
200
201struct PendingJoinExchange {
202 peer: [u8; 32],
203 task: tokio::task::JoinHandle<JoinAttemptOutcome>,
204}
205impl Drop for PendingJoinExchange {
206 fn drop(&mut self) {
207 self.task.abort();
208 }
209}
210
211struct PendingCheckpointExchange {
212 peer: [u8; 32],
213 task: tokio::task::JoinHandle<Result<Value, String>>,
214}
215impl Drop for PendingCheckpointExchange {
216 fn drop(&mut self) {
217 self.task.abort();
218 }
219}
220
221struct ReadyRange {
222 query: arachne_delivery::RangeQuery,
223 peer: [u8; 32],
224 reply: Vec<u8>,
225 packet_count: usize,
226 automatic: bool,
227}
228
229struct ReadyDirectRange {
230 query: arachne_delivery::wire::DirectRangeQuery,
231 peer: [u8; 32],
232 reply: Vec<u8>,
233 packet_count: usize,
234}
235
236struct ReadyCurrentView {
237 query: arachne_delivery::current::CurrentViewQuery,
238 peer: [u8; 32],
239 reply: Vec<u8>,
240 cut: u64,
241 value_count: usize,
242}
243
244type RecoveryReply = ([u8; 32], Result<Vec<u8>, String>);
245
246struct PendingRange {
247 query: arachne_delivery::RangeQuery,
248 available: Option<arachne_delivery::wire::AvailableRangeQuery>,
249 automatic: bool,
250 replies: mpsc::Receiver<RecoveryReply>,
251 task: tokio::task::JoinHandle<()>,
252 attempted: usize,
253 reason: Option<String>,
254}
255impl Drop for PendingRange {
256 fn drop(&mut self) {
257 self.task.abort();
258 }
259}
260
261struct PendingDirectRange {
262 query: arachne_delivery::wire::DirectRangeQuery,
263 replies: mpsc::Receiver<RecoveryReply>,
264 task: tokio::task::JoinHandle<()>,
265 attempted: usize,
266 reason: Option<String>,
267}
268impl Drop for PendingDirectRange {
269 fn drop(&mut self) {
270 self.task.abort();
271 }
272}
273
274struct PendingCurrentView {
275 query: arachne_delivery::current::CurrentViewQuery,
276 automatic: bool,
277 replies: mpsc::Receiver<RecoveryReply>,
278 task: tokio::task::JoinHandle<()>,
279 attempted: usize,
280 reason: Option<String>,
281 best: Option<ReadyCurrentView>,
282}
283impl Drop for PendingCurrentView {
284 fn drop(&mut self) {
285 self.task.abort();
286 }
287}
288
289struct Session {
290 resources: resources::Jobs,
291 presence: presence::Presence,
292 interests: interest::Updates,
293 records: Option<persistence::NativeStore>,
294 membership_update: Option<PendingControl<membership::StateBasis>>,
295 membership_offer: Option<PendingControl<u64>>,
296 membership_offer_requires_adoption: bool,
297 membership_peer_failures: BTreeMap<[u8; 32], std::time::Instant>,
299 staged_step_received: bool,
301 gossip_steps_ahead: BTreeMap<u64, Vec<u8>>,
304 membership_head: Option<(u64, Vec<[u8; 32]>)>,
307 range_pull: Option<PendingControl<u64>>,
309 gossip_counts: Arc<membership::GossipCounts>,
311 profile_pull: Option<PendingControl<membership::ProfilePull>>,
314 profiles_walked: BTreeMap<[u8; 32], [u8; 32]>,
315 gossip_profiles_pending: std::collections::VecDeque<Vec<u8>>,
317 cutoff: Option<PendingControl<arachne_delivery::wire::CutoffQuery>>,
318 current_view: Option<PendingCurrentView>,
319 ready_current_view: Option<ReadyCurrentView>,
320 range: Option<PendingRange>,
321 ready_range: Option<ReadyRange>,
322 direct_range: Option<PendingDirectRange>,
323 ready_direct_range: Option<ReadyDirectRange>,
324 direct_miss: Option<arachne_delivery::wire::DirectRangeQuery>,
325 recovered: VecDeque<(
326 arachne_routing::PublicationContext,
327 arachne_security::ApplicationMessage,
328 )>,
329 publisher: Option<arachne_delivery::PublisherLog>,
330 received: Option<arachne_delivery::receive::ReceiveJournal>,
331 inbox: Option<arachne_delivery::inbox::ObjectInbox>,
332 inbound_admission: Option<arachne_node::ControlRequest>,
333 admission_queue: arachne_security::AdmissionQueue,
334 admission_metadata: BTreeMap<[u8; 32], QueuedAdmission>,
335 admission_waiters: admission_waiters::AdmissionWaiters<arachne_node::ControlRequest>,
336 admission_pushes: Vec<PendingControl<[u8; 32]>>,
337 pending_approvals: BTreeMap<[u8; 32], PendingAdmissionApproval>,
338 staged_approval_id: Option<[u8; 32]>,
339 queued_admission_in_flight: Vec<arachne_security::AdmissionAttempt>,
342 admission_reads_since_stage: usize,
346 nearby_workspaces: BTreeMap<[u8; 32], Vec<u8>>,
347 nearby_identity: Option<String>,
348 staged_workspace: Option<StagedWorkspace>,
349 profiles: membership::Profiles,
351 peer_profile_summaries: BTreeMap<[u8; 32], [u8; 32]>,
352 staged_removal: Option<(arachne_security::RemovedMembership, Vec<u8>)>,
353 workspace: Option<Arc<arachne_security::Workspace>>,
356 committed: committed_view::Published,
358 pending_join: Option<arachne_security::PendingJoin>,
359 join_lifecycle: Option<JoinLifecycle>,
360 checkpoint_exchange: Option<PendingCheckpointExchange>,
361 join_exchange: Option<PendingJoinExchange>,
362 activity: WorkspaceActivity,
363 join_history_prefix: Vec<Value>,
367 storage_key: Option<arachne_security::StorageKey>,
368 overlay_paths: usize,
369 node: Node,
370 receiver: arachne_node::MessageReceiver,
371 runtime: Runtime,
372}
373
374struct Registry {
375 next: i64,
376 sessions: BTreeMap<i64, SharedSession>,
377 signals: BTreeMap<i64, Arc<work_signal::WorkSignal>>,
379 cancellations: BTreeMap<i64, watch::Sender<bool>>,
380 connection_budget: ConnectionBudget,
381}
382
383type SharedSession = Arc<Mutex<Option<Session>>>;
384
385static REGISTRY: std::sync::LazyLock<Mutex<Registry>> = std::sync::LazyLock::new(|| {
389 Mutex::new(Registry {
390 next: 1,
391 sessions: BTreeMap::new(),
392 signals: BTreeMap::new(),
393 cancellations: BTreeMap::new(),
394 connection_budget: ConnectionBudget::default(),
395 })
396});
397const MAX_DEVICE_OVERLAY_PATHS: usize = 24;
398const MAX_WORKSPACE_OVERLAY_PATHS: usize = 5;
399const NEARBY_INVITATION: &[u8; 5] = b"DFNI\x01";
400const NEARBY_WORKSPACE: &[u8; 5] = b"DFNW\x01";
401const NEARBY_WORKSPACE_LIST: &[u8; 5] = b"DFNW\x02";
402const NEARBY_IDENTITY: &[u8; 5] = b"DFND\x01";
403const ADMISSION_RESULT_OFFER: &[u8; 5] = b"DFAR\x01";
404const MAX_NEARBY_WORKSPACES: usize = 16;
405const MAX_RUNTIME_ADMISSION_BATCH: usize = 16;
409const MAX_ADMISSION_WAITERS: usize = 256;
412static DEVICE_OVERLAY_PATHS: AtomicUsize = AtomicUsize::new(0);
413
414pub mod admission_state {
416 pub const QUEUED: &str = "admission_queued";
417 pub const APPROVAL_PENDING: &str = "approval_pending";
418 pub const APPROVAL_REQUESTED: &str = "approval_requested";
419 pub const REPLIED: &str = "admission_replied";
420 pub const UNAVAILABLE: &str = "admission_unavailable";
421 pub const RECOVERY_REQUIRED: &str = "admission_recovery_required";
422 pub const MEMBER_ALREADY_ADMITTED: &str = "member_already_admitted";
423 pub const RECOVERY_REMOVE_AND_REINVITE: &str = "remove_and_reinvite";
424 pub const NOT_SENT: &str = "admission_not_sent";
425 pub const WAITING: &str = "admission_waiting";
428}
429
430fn transition_activity(
431 session: &mut Session,
432 phase: WorkspacePhase,
433 reason: Option<&str>,
434) -> Result<(), String> {
435 session.activity.transition(phase, reason)
436}
437
438fn activity_value(session: &Session) -> Value {
439 session.activity.projection()
440}
441
442fn nearby_workspace_key(workspace: Option<[u8; 32]>, invitation: &[u8]) -> [u8; 32] {
443 workspace
444 .or_else(|| {
445 arachne_security::Invitation::from_bytes(invitation)
446 .ok()
447 .map(|value| value.workspace_id())
448 })
449 .unwrap_or_else(|| Sha256::digest(invitation).into())
450}
451
452fn nearby_workspace_reply(advertisements: &BTreeMap<[u8; 32], Vec<u8>>) -> Vec<u8> {
453 match advertisements.len() {
454 0 => Vec::new(),
455 1 => advertisements.values().next().cloned().unwrap_or_default(),
456 count => {
457 let mut reply = NEARBY_WORKSPACE_LIST.to_vec();
458 reply.push(count as u8);
459 for payload in advertisements.values() {
460 reply.extend((payload.len() as u16).to_be_bytes());
461 reply.extend(payload);
462 }
463 reply
464 }
465 }
466}
467
468fn nearby_workspace_payloads(reply: &[u8]) -> Vec<&[u8]> {
469 if reply.is_empty() {
470 return Vec::new();
471 }
472 if !reply.starts_with(NEARBY_WORKSPACE_LIST) {
473 return vec![reply];
474 }
475 let Some(&count) = reply.get(NEARBY_WORKSPACE_LIST.len()) else {
476 return Vec::new();
477 };
478 if count == 0 || count as usize > MAX_NEARBY_WORKSPACES {
479 return Vec::new();
480 }
481 let mut offset = NEARBY_WORKSPACE_LIST.len() + 1;
482 let mut payloads = Vec::with_capacity(count as usize);
483 for _ in 0..count {
484 let Some(length) = reply.get(offset..offset + 2) else {
485 return Vec::new();
486 };
487 let length = u16::from_be_bytes([length[0], length[1]]) as usize;
488 offset += 2;
489 if length == 0 || length > 4096 || offset + length > reply.len() {
490 return Vec::new();
491 }
492 payloads.push(&reply[offset..offset + length]);
493 offset += length;
494 }
495 if offset != reply.len() {
496 return Vec::new();
497 }
498 payloads
499}
500
501pub fn create(secret: Option<&[u8; 32]>) -> Result<i64, String> {
504 create_endpoint(secret, false, false, false, false, None)
505}
506
507pub fn create_lan(secret: &[u8; 32]) -> Result<i64, String> {
510 create_endpoint(Some(secret), true, false, false, false, None)
511}
512
513pub fn create_nearby(secret: &[u8; 32]) -> Result<i64, String> {
515 create_endpoint(Some(secret), true, false, false, true, None)
516}
517
518pub fn create_wan(secret: &[u8; 32]) -> Result<i64, String> {
521 create_endpoint(Some(secret), true, true, false, false, None)
522}
523
524pub fn create_relay(secret: &[u8; 32]) -> Result<i64, String> {
527 create_endpoint(Some(secret), false, true, true, false, None)
528}
529
530pub fn create_relay_with_options(secret: &[u8; 32], relay: RelayOptions) -> Result<i64, String> {
533 create_endpoint(Some(secret), false, true, true, false, Some(relay))
534}
535
536pub fn create_wan_only(secret: &[u8; 32]) -> Result<i64, String> {
539 create_endpoint(Some(secret), false, true, false, false, None)
540}
541
542fn create_endpoint(
543 secret: Option<&[u8; 32]>,
544 lan_lookup: bool,
545 wan_lookup: bool,
546 relay_only: bool,
547 nearby: bool,
548 relay: Option<RelayOptions>,
549) -> Result<i64, String> {
550 let mut registry = REGISTRY.lock().map_err(|_| "node registry unavailable")?;
551 if registry.sessions.len() >= 8 || registry.next == i64::MAX {
552 return Err("node limit reached".into());
553 }
554 let connection_budget = registry.connection_budget.clone();
555 let runtime = tokio::runtime::Builder::new_multi_thread()
556 .worker_threads(2)
557 .enable_all()
558 .build()
559 .map_err(|e| e.to_string())?;
560 let (node, receiver) = runtime
561 .block_on(async {
562 tokio::time::timeout(Duration::from_secs(10), async {
563 let address = ([0, 0, 0, 0], 0).into();
564 let profile = match (secret, lan_lookup, wan_lookup, relay_only, nearby) {
565 (Some(_), _, _, _, true) => NetworkProfile::Nearby,
566 (Some(_), _, _, true, false) => NetworkProfile::RelayOnly,
567 (Some(_), false, true, false, false) => NetworkProfile::WanOnly,
568 (Some(_), _, true, false, false) => NetworkProfile::Wan,
569 (Some(_), true, false, false, false) => NetworkProfile::Lan,
570 _ => NetworkProfile::Direct,
571 };
572 let bound = match relay {
573 Some(relay) => {
574 Node::bind_with_profile_and_relay(
575 address,
576 secret,
577 profile,
578 connection_budget,
579 relay,
580 )
581 .await?
582 }
583 None => {
584 Node::bind_with_profile(address, secret, profile, connection_budget).await?
585 }
586 };
587 if relay_only {
588 bound.0.wait_online().await;
589 }
590 Ok::<_, arachne_node::Error>(bound)
591 })
592 .await
593 })
594 .map_err(|_| "node startup timed out")?
595 .map_err(|e| e.to_string())?;
596 let handle = registry.next;
597 registry.next += 1;
598 let signal = Arc::new(work_signal::WorkSignal::default());
599 let committed = committed_view::Published::new(Some(Arc::clone(&signal)));
600 node.set_inquiry_responder(committed.responder());
601 let arrivals = node.control_signal();
602 let forward = Arc::clone(&signal);
603 runtime.spawn(async move {
605 loop {
606 arrivals.notified().await;
607 forward.raise();
608 }
609 });
610 registry.signals.insert(handle, signal);
611 registry
612 .cancellations
613 .insert(handle, node.control_cancellation());
614 let presence = presence::Presence::new()?;
615 registry.sessions.insert(
616 handle,
617 Arc::new(Mutex::new(Some(Session {
618 resources: resources::Jobs::default(),
619 presence,
620 interests: interest::Updates::default(),
621 records: None,
622 membership_update: None,
623 membership_offer: None,
624 membership_offer_requires_adoption: false,
625 membership_peer_failures: BTreeMap::new(),
626 staged_step_received: false,
627 gossip_steps_ahead: BTreeMap::new(),
628 membership_head: None,
629 range_pull: None,
630 gossip_counts: Arc::default(),
631 profile_pull: None,
632 profiles_walked: BTreeMap::new(),
633 gossip_profiles_pending: Default::default(),
634 cutoff: None,
635 current_view: None,
636 ready_current_view: None,
637 range: None,
638 ready_range: None,
639 direct_range: None,
640 ready_direct_range: None,
641 direct_miss: None,
642 recovered: VecDeque::new(),
643 publisher: None,
644 received: None,
645 inbox: None,
646 inbound_admission: None,
647 admission_queue: arachne_security::AdmissionQueue::new(),
648 admission_metadata: BTreeMap::new(),
649 admission_waiters: admission_waiters::AdmissionWaiters::new(MAX_ADMISSION_WAITERS),
650 admission_pushes: Vec::new(),
651 pending_approvals: BTreeMap::new(),
652 staged_approval_id: None,
653 queued_admission_in_flight: Vec::new(),
654 admission_reads_since_stage: 0,
655 nearby_workspaces: BTreeMap::new(),
656 nearby_identity: None,
657 staged_workspace: None,
658 staged_removal: None,
659 profiles: committed.profiles(),
660 peer_profile_summaries: BTreeMap::new(),
661 workspace: None,
662 committed,
663 pending_join: None,
664 join_lifecycle: None,
665 checkpoint_exchange: None,
666 join_exchange: None,
667 activity: WorkspaceActivity::default(),
668 join_history_prefix: Vec::new(),
669 storage_key: secret
670 .map(arachne_security::StorageKey::derive)
671 .transpose()
672 .map_err(str::to_owned)?,
673 overlay_paths: 0,
674 node,
675 receiver,
676 runtime,
677 }))),
678 );
679 Ok(handle)
680}
681
682fn commit_workspace(session: &mut Session, workspace: arachne_security::Workspace) {
685 let workspace = Arc::new(workspace);
686 session
687 .committed
688 .publish(workspace.clone(), session.node.id());
689 session.workspace = Some(workspace);
690 session.presence.announce_next();
694 membership::retain_held_profiles(session);
696}
697
698fn session(handle: i64) -> Result<SharedSession, String> {
699 REGISTRY
700 .lock()
701 .map_err(|_| "node registry unavailable")?
702 .sessions
703 .get(&handle)
704 .cloned()
705 .ok_or_else(|| "invalid or closed node handle".into())
706}
707
708pub fn describe(handle: i64) -> Result<String, String> {
710 let shared = session(handle)?;
711 let guard = shared.lock().map_err(|_| "node session unavailable")?;
712 let session = guard.as_ref().ok_or("node is closed")?;
713 Ok(json!({
714 "endpoint_key": session.node.id(),
715 "bound_address": session.node.address().to_string(),
716 "workspace_ready": session.workspace.is_some(),
717 "activity": activity_value(session),
718 })
719 .to_string())
720}
721
722pub fn close(handle: i64) -> Result<(), String> {
724 let (shared, signal, cancellation) = {
725 let mut registry = REGISTRY.lock().map_err(|_| "node registry unavailable")?;
726 let shared = registry
727 .sessions
728 .remove(&handle)
729 .ok_or("invalid or closed node handle")?;
730 (
731 shared,
732 registry.signals.remove(&handle),
733 registry.cancellations.remove(&handle),
734 )
735 };
736 if let Some(cancellation) = cancellation {
737 cancellation.send_replace(true);
738 }
739 if let Some(signal) = signal {
740 signal.close();
741 }
742 let session = shared
744 .lock()
745 .map_err(|_| "node session unavailable")?
746 .take();
747 session.map(shutdown_session).unwrap_or(Ok(()))
750}
751
752pub fn cancel(handle: i64) -> Result<(), String> {
755 let cancellation = REGISTRY
756 .lock()
757 .map_err(|_| "node registry unavailable")?
758 .cancellations
759 .get(&handle)
760 .cloned()
761 .ok_or("invalid or closed node handle")?;
762 cancellation.send_replace(true);
763 Ok(())
764}
765
766pub fn wait_for_work(handle: i64) -> Result<bool, String> {
770 let signal = REGISTRY
771 .lock()
772 .map_err(|_| "node registry unavailable")?
773 .signals
774 .get(&handle)
775 .cloned()
776 .ok_or("invalid or closed node handle")?;
777 Ok(signal.wait())
778}
779
780fn shutdown_session(mut session: Session) -> Result<(), String> {
781 release_overlay_paths(&DEVICE_OVERLAY_PATHS, session.overlay_paths);
782 session.overlay_paths = 0;
783 session.presence.cancel();
784 session.interests.cancel();
785 drop(session.join_exchange.take());
786 drop(session.checkpoint_exchange.take());
787 drop(session.cutoff.take());
788 drop(session.current_view.take());
789 session.ready_current_view = None;
790 drop(session.range.take());
791 session.resources = resources::Jobs::default();
792 let result = session.runtime.block_on(async {
793 tokio::time::timeout(Duration::from_secs(5), session.node.close()).await
794 });
795 session.runtime.shutdown_timeout(Duration::from_secs(2));
796 result.map_err(|_| "node shutdown timed out".into())
797}
798
799fn reserve_overlay_paths(total: &AtomicUsize, additional: usize) -> bool {
800 total
801 .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| {
802 current
803 .checked_add(additional)
804 .filter(|next| *next <= MAX_DEVICE_OVERLAY_PATHS)
805 })
806 .is_ok()
807}
808
809fn release_overlay_paths(total: &AtomicUsize, count: usize) {
810 if count != 0 {
811 let previous = total.fetch_sub(count, Ordering::AcqRel);
812 debug_assert!(previous >= count);
813 }
814}
815
816async fn install_gossip_policy(
817 node: &Node,
818 reserved: &mut usize,
819 workspace: [u8; 32],
820 revision: u64,
821 policy: BTreeMap<[u8; 32], Permissions>,
822) -> Result<(), String> {
823 let desired = policy
824 .keys()
825 .filter(|peer| **peer != node.id())
826 .count()
827 .min(MAX_WORKSPACE_OVERLAY_PATHS);
828 let additional = desired.saturating_sub(*reserved);
829 if !reserve_overlay_paths(&DEVICE_OVERLAY_PATHS, additional) {
830 return Err("device overlay path limit reached".into());
831 }
832 let result = async {
833 node.install_verified_policy(workspace, revision, policy)
834 .await
835 .map_err(|error| error.to_string())?;
836 node.enable_gossip(workspace, revision)
837 .await
838 .map_err(|error| error.to_string())
839 }
840 .await;
841 if let Err(error) = result {
842 release_overlay_paths(&DEVICE_OVERLAY_PATHS, additional);
843 return Err(error);
844 }
845 if *reserved > desired {
846 release_overlay_paths(&DEVICE_OVERLAY_PATHS, *reserved - desired);
847 }
848 *reserved = desired;
849 Ok(())
850}
851
852const RANGE_SCAN_DEPTH: usize = 64;
854
855pub const MAX_REQUEST: usize = 128 * 1024;
857
858#[derive(Clone, Copy, Deserialize)]
859#[serde(deny_unknown_fields)]
860struct CurrentPublication {
861 selector: [u8; 32],
862 replacement_key: [u8; 32],
863 expires_at: u64,
864 #[serde(default)]
865 tombstone: bool,
866}
867
868#[derive(Deserialize)]
869#[serde(tag = "op", rename_all = "snake_case", deny_unknown_fields)]
870enum Request {
871 Resource {
872 request: resources::Request,
873 },
874 WorkspaceMetrics {},
875 WorkspaceState {},
876 ResetWorkspace {},
877 DiscardWorkspaceCandidate {},
878 NetworkChange {},
879 NearbyEndpoints {},
880 SetNearbyIdentity {
881 name: String,
882 },
883 NearbyWorkspaces {},
884 SetNearbyWorkspace {
885 mode: Option<String>,
886 #[serde(default)]
887 invitation: Vec<u8>,
888 #[serde(default)]
889 workspace_name: Option<String>,
890 #[serde(default)]
891 workspace: Option<[u8; 32]>,
892 },
893 SendNearbyInvitation {
894 peer: [u8; 32],
895 invitation: Vec<u8>,
896 },
897 PollWorkspacePresence {
898 #[serde(default)]
899 announce: bool,
900 },
901 FetchMembershipUpdate {
902 peer: [u8; 32],
903 #[serde(default)]
904 replace_pending: bool,
905 },
906 PollMembershipUpdate {},
907 NextMembershipPeer {
908 after: Option<[u8; 32]>,
909 },
910 OfferMembershipUpdate {
911 peer: [u8; 32],
912 after: u64,
913 },
914 OfferStagedMembershipUpdate {
915 peer: [u8; 32],
916 },
917 PollMembershipOffer {},
918 FetchRecoveryRange {
919 #[serde(default)]
920 peer: Option<[u8; 32]>,
921 #[serde(default)]
922 author: Option<[u8; 32]>,
923 revision: u64,
924 topics: Vec<String>,
925 #[serde(default)]
926 after: Option<u64>,
927 #[serde(default)]
928 through: Option<u64>,
929 },
930 PollRecoveryRange {},
931 NextDirectGap {},
932 FetchDirectRecovery {
933 author: [u8; 32],
934 revision: u64,
935 topic: String,
936 recipients: Vec<[u8; 32]>,
937 after: u64,
938 through: u64,
939 },
940 PollDirectRecovery {},
941 StageDirectRecovery {},
942 StageDirectMiss {},
943 CancelDirectRecovery {},
944 StageRecoveryRange {
945 #[serde(default)]
946 retain_until: u64,
947 },
948 AdoptRecovery {
949 #[serde(default)]
950 snapshot: Vec<u8>,
951 },
952 PollRecoveredPublication {},
953 CancelRecoveryRange {},
954 PollRecoveryCutoff {},
955 DiscoverRecoveryCutoff {
956 peer: [u8; 32],
957 revision: u64,
958 topics: Vec<String>,
959 },
960 FetchCurrentView {
961 #[serde(default)]
962 peer: Option<[u8; 32]>,
963 authority: [u8; 32],
964 revision: u64,
965 topic: String,
966 selector: [u8; 32],
967 },
968 PollCurrentView {},
969 StageCurrentView {},
970 AdoptCurrentView {
971 #[serde(default)]
972 snapshot: Vec<u8>,
973 },
974 CancelCurrentView {},
975 FetchInvitationCheckpoint {
976 #[serde(default)]
977 peer: Option<[u8; 32]>,
978 #[serde(default)]
979 peers: Vec<[u8; 32]>,
980 invitation: Vec<u8>,
981 },
982 #[serde(rename = "request_admission")]
983 JoinViaPeer {
984 peer: [u8; 32],
985 },
986 StageNetworkPublication {
987 revision: u64,
988 topic: String,
989 id: [u8; 16],
990 payload: Vec<u8>,
991 #[serde(default)]
994 recipients: Vec<[u8; 32]>,
995 #[serde(default)]
996 current: Option<CurrentPublication>,
997 #[serde(default)]
998 bulk: bool,
999 },
1000 EnableObjectDelivery {},
1001 PollPendingObject {
1002 #[serde(default)]
1003 deferred: Vec<arachne_delivery::inbox::DeferredDeliveryStream>,
1004 },
1005 StageObjectAcknowledgement {
1006 member: [u8; 32],
1007 topic: String,
1008 counter: u64,
1009 id: [u8; 16],
1010 },
1011 StageObjectRejection {
1012 member: [u8; 32],
1013 topic: String,
1014 counter: u64,
1015 id: [u8; 16],
1016 },
1017 PollProtected {},
1018 EndpointInfo {},
1019 ControlExchange {
1023 peer: [u8; 32],
1024 address: Option<String>,
1025 payload: Vec<u8>,
1026 },
1027 StagePublication {
1028 context: Vec<u8>,
1029 payload: Vec<u8>,
1030 },
1031 AdoptPublication {
1032 #[serde(default)]
1033 snapshot: Vec<u8>,
1034 },
1035 StageReception {
1036 context: Vec<u8>,
1037 ciphertext: Vec<u8>,
1038 },
1039 AdoptReception {
1040 #[serde(default)]
1041 snapshot: Vec<u8>,
1042 },
1043 PollAdmission {
1044 #[serde(default)]
1048 profile: bool,
1049 },
1050 DriveWorkspace {},
1054 ListAdmissionApprovals {
1055 #[serde(default)]
1056 after: Option<[u8; 32]>,
1057 #[serde(default)]
1058 limit: Option<usize>,
1059 },
1060 AcknowledgeAdmissionApproval {
1061 attempt_id: [u8; 32],
1062 },
1063 SendAdmissionReply {},
1064 StageJoin {
1065 commits: Vec<JoinStep>,
1066 #[serde(default)]
1067 welcome: Vec<u8>,
1068 },
1069 AdoptJoin {
1070 #[serde(default)]
1071 snapshot: Vec<u8>,
1072 },
1073 StageAdmission {
1075 authenticated_endpoint: [u8; 32],
1076 request: Vec<u8>,
1077 },
1078 MemberRoster {
1079 #[serde(default)]
1080 profiles: Vec<Vec<u8>>,
1081 },
1082 UseServiceProfile {},
1083 LeaveViaPeer {
1084 peer: [u8; 32],
1085 },
1086 StageSoloLeave {},
1087 StageManagement {
1088 action: WireManagement,
1089 },
1090 StageInvitation {
1091 expires_at: u64,
1092 personal: bool,
1093 #[serde(default)]
1094 automatic: bool,
1095 #[serde(default)]
1096 request_access: bool,
1097 },
1098 StageInvitationApproval {
1099 request: Vec<u8>,
1100 #[serde(default)]
1101 attempt_id: Option<[u8; 32]>,
1102 },
1103 StageInvitationDecline {
1104 request: Vec<u8>,
1105 #[serde(default)]
1106 attempt_id: Option<[u8; 32]>,
1107 },
1108 InvitationControls {},
1109 StageWorkspaceName {
1110 workspace_name: String,
1111 },
1112 StageWorkspaceNameUpdate {
1113 name_record: Vec<u8>,
1114 },
1115 StageWorkspaceNameCheckpoint {
1116 name_checkpoint: Vec<u8>,
1117 },
1118 InspectInvitation {
1119 invitation: Vec<u8>,
1120 checkpoint: Vec<u8>,
1121 },
1122 StageAdmissionUpdate {
1123 step: JoinStep,
1124 },
1125 AdoptAdmission {
1126 #[serde(default)]
1127 snapshot: Vec<u8>,
1128 },
1129 RetainedAdmission {
1130 authenticated_endpoint: [u8; 32],
1131 request: Vec<u8>,
1132 },
1133 IssueInvitation {},
1134 BeginJoin {
1135 invitation: Vec<u8>,
1136 #[serde(default)]
1137 checkpoint: Vec<u8>,
1138 display_name: String,
1139 #[serde(default)]
1140 peers: Vec<[u8; 32]>,
1141 },
1142 DriveJoin {},
1143 SealPendingJoin {},
1144 RestorePendingJoin {
1145 workspace: [u8; 32],
1146 #[serde(default)]
1147 snapshot: Vec<u8>,
1148 },
1149 CreateWorkspace {
1150 display_name: String,
1151 workspace_name: Option<String>,
1152 },
1153 SealWorkspace {},
1154 RestoreWorkspace {
1155 workspace: [u8; 32],
1156 #[serde(default)]
1157 snapshot: Vec<u8>,
1158 },
1159 AddAddressHint {
1160 peer: [u8; 32],
1161 address: String,
1162 },
1163 InstallWorkspacePolicy {
1165 revision: u64,
1166 },
1167 InstallMemberPolicy {
1168 revision: u64,
1169 topics: Vec<String>,
1170 },
1171 InstallVerifiedPolicy {
1173 workspace: [u8; 32],
1174 revision: u64,
1175 endpoints: Vec<EndpointPolicy>,
1176 },
1177 SetInterest {
1178 workspace: [u8; 32],
1179 revision: u64,
1180 topic: String,
1181 subscribed: bool,
1182 },
1183 PollInterest {},
1184 Subscribe {
1185 workspace: [u8; 32],
1186 revision: u64,
1187 topic: String,
1188 },
1189 Unsubscribe {
1190 workspace: [u8; 32],
1191 revision: u64,
1192 topic: String,
1193 },
1194 Publish {
1195 workspace: [u8; 32],
1196 revision: u64,
1197 topic: String,
1198 payload: Vec<u8>,
1199 },
1200 Poll {},
1201}
1202
1203use membership::{JoinStep, WireManagement};
1204
1205#[derive(Deserialize)]
1206#[serde(deny_unknown_fields)]
1207struct EndpointPolicy {
1208 peer: [u8; 32],
1209 publish: Vec<String>,
1210 subscribe: Vec<String>,
1211}
1212
1213fn report(value: AdmissionReport) -> Value {
1214 json!({ "admitted": value.admitted,
1215 "queued": value.queued,
1216 "failed": value.failed.into_iter().map(|(peer, error)|
1217 json!({"peer": peer, "error": error.to_string()})).collect::<Vec<_>>() })
1218}
1219
1220#[derive(Deserialize)]
1221#[serde(deny_unknown_fields)]
1222struct InvitationInspection {
1223 invitation: Vec<u8>,
1224 checkpoint: Vec<u8>,
1225}
1226
1227pub fn inspect_invitation(request: &[u8]) -> Result<Vec<u8>, String> {
1229 if request.len() > MAX_REQUEST {
1230 return Err("request exceeds limit".into());
1231 }
1232 let request: InvitationInspection =
1233 serde_json::from_slice(request).map_err(|e| e.to_string())?;
1234 serde_json::to_vec(&inspected_invitation(
1235 &request.invitation,
1236 &request.checkpoint,
1237 )?)
1238 .map_err(|e| e.to_string())
1239}
1240
1241fn invitation_envelope(
1242 session: &Session,
1243 invitation: &arachne_security::Invitation,
1244 checkpoint: Vec<u8>,
1245) -> Result<Value, String> {
1246 let owner = session
1247 .workspace
1248 .as_ref()
1249 .ok_or("session has no workspace")?;
1250 let mut members = owner.member_endpoints().map_err(str::to_owned)?;
1251 members.sort_unstable();
1252 let bootstrap_peers = std::iter::once(session.node.id())
1253 .chain(
1254 members
1255 .iter()
1256 .copied()
1257 .filter(|peer| *peer != session.node.id()),
1258 )
1259 .take(3)
1260 .collect::<Vec<_>>();
1261 let mut routes = Vec::new();
1262 for peer in members {
1263 if peer != session.node.id()
1264 && let Some(address) = session.runtime.block_on(session.node.address_hint(peer))
1265 && address.is_ipv4()
1266 && !address.ip().is_unspecified()
1267 {
1268 routes.push(json!({"peer":peer,"address":address.to_string()}));
1269 if routes.len() == 7 {
1270 break;
1271 }
1272 }
1273 }
1274 Ok(
1275 json!({"workspace": owner.id(), "workspace_name":owner.workspace_name().map_err(str::to_owned)?, "invitation": invitation.export_secret_token().as_slice(), "invitation_key": invitation.key(), "checkpoint": checkpoint, "peer":session.node.id(), "bootstrap_peers":bootstrap_peers, "address":session.node.address().to_string(), "routes":routes}),
1276 )
1277}
1278
1279fn inspected_invitation(invitation: &[u8], checkpoint: &[u8]) -> Result<Value, String> {
1280 let invitation = arachne_security::Invitation::from_bytes(invitation).map_err(str::to_owned)?;
1281 let proof = invitation.join_proof(checkpoint).map_err(str::to_owned)?;
1282 let control = proof
1283 .invitation_control(invitation.key())
1284 .map_err(str::to_owned)?;
1285 Ok(
1286 json!({"workspace":invitation.workspace_id(),"invitation_key":invitation.key(),"workspace_name":proof.workspace_name().map_err(str::to_owned)?,"epoch":proof.epoch(), "personal_invitation":control.as_ref().is_some_and(|c| c.personal), "automatic_approval":control.as_ref().is_some_and(|c| c.automatic()), "expires_at":control.map_or(0, |c| c.expires_at)}),
1287 )
1288}
1289
1290fn member_metadata(workspace: &arachne_security::Workspace) -> Value {
1291 workspace.member().map_or(
1292 Value::Null,
1293 |member| json!({"id": member.id(), "display_name": member.display_name()}),
1294 )
1295}
1296
1297fn pending_metadata(
1298 pending: &arachne_security::PendingJoin,
1299 endpoint: [u8; 32],
1300) -> Result<Value, String> {
1301 Ok(
1302 json!({"workspace": pending.workspace_id(), "workspace_name":pending.workspace_name().map_err(str::to_owned)?, "endpoint":endpoint, "state": "pending", "durable": false,
1303 "member": {"id": pending.member().id(), "display_name": pending.member().display_name()},
1304 "key_package": pending.key_package().map_err(str::to_owned)?,
1305 "personal_invitation":if pending.admission_request().is_ok() { pending.personal_invitation().map_err(str::to_owned)? } else { false }, "admission_request": pending.admission_request().ok()}),
1306 )
1307}
1308
1309fn check_recovery_policy(
1310 session: &Session,
1311 peer: [u8; 32],
1312 author: [u8; 32],
1313 workspace: [u8; 32],
1314 epoch: u64,
1315 revision: u64,
1316 topics: &BTreeSet<Topic>,
1317) -> Result<(), String> {
1318 let owner = session
1319 .workspace
1320 .as_ref()
1321 .ok_or("session has no workspace")?;
1322 if owner.id() != workspace || owner.epoch() != epoch {
1323 return Err("recovery scope changed".into());
1324 }
1325 if !owner
1326 .member_endpoints()
1327 .map_err(str::to_owned)?
1328 .contains(&peer)
1329 {
1330 return Err("recovery peer is not a current member".into());
1331 }
1332 let author_endpoint = owner
1333 .endpoints_for_members(&[author])
1334 .map_err(str::to_owned)?[0];
1335 session
1336 .runtime
1337 .block_on(session.node.with_routing_policy(|policy| {
1338 for topic in topics {
1339 let publishers = policy
1340 .publishers(owner.id(), revision, session.node.id(), topic)
1341 .map_err(|e| e.to_string())?;
1342 if !publishers.contains(&author_endpoint) {
1343 return Err("recovery publisher is not authorized".to_owned());
1344 }
1345 if peer != author_endpoint
1346 && !policy
1347 .publishers(owner.id(), revision, peer, topic)
1348 .map_err(|e| e.to_string())?
1349 .contains(&author_endpoint)
1350 {
1351 return Err("recovery holder is not authorized".to_owned());
1352 }
1353 }
1354 Ok(())
1355 }))
1356}
1357
1358fn seal_state(
1359 native: bool,
1360 workspace: &arachne_security::Workspace,
1361 key: &arachne_security::StorageKey,
1362 publisher: Option<&arachne_delivery::PublisherLog>,
1363 received: Option<&arachne_delivery::receive::ReceiveJournal>,
1364 inbox: Option<&arachne_delivery::inbox::ObjectInbox>,
1365) -> Result<Vec<u8>, String> {
1366 if native {
1367 return persistence::candidate_token();
1368 }
1369 if let Some(inbox) = inbox {
1370 return inbox
1371 .with_legacy_receipts(received)
1372 .seal(
1373 workspace,
1374 key,
1375 publisher.ok_or("object inbox requires publisher state")?,
1376 )
1377 .map_err(str::to_owned);
1378 }
1379 match (publisher, received) {
1380 (Some(log), Some(received)) => log.seal_with_receipts(workspace, key, received),
1381 (Some(log), None) => log.seal(workspace, key),
1382 (None, None) => workspace.seal(key),
1383 (None, Some(_)) => return Err("receive journal requires publisher state".into()),
1384 }
1385 .map_err(str::to_owned)
1386}
1387
1388fn check_epoch_transition(session: &mut Session) -> Result<(), String> {
1390 let owner = session
1391 .workspace
1392 .as_ref()
1393 .ok_or("session has no workspace")?;
1394 if let Some(inbox) = &session.inbox {
1395 inbox.pending(owner).map_err(str::to_owned)?;
1397 if inbox.pending_count() > 0 {
1398 return Err(
1399 "pending application delivery must be acknowledged before membership update".into(),
1400 );
1401 }
1402 }
1403 if !session.recovered.is_empty() {
1404 return Err("recovery must finish or cancel before membership update".into());
1405 }
1406 drop(session.cutoff.take());
1411 drop(session.range.take());
1412 session.ready_range = None;
1413 drop(session.direct_range.take());
1414 session.ready_direct_range = None;
1415 session.direct_miss = None;
1416 drop(session.current_view.take());
1417 session.ready_current_view = None;
1418 Ok(())
1419}
1420
1421fn stage_admission(
1422 session: &mut Session,
1423 authenticated_endpoint: [u8; 32],
1424 request: &[u8],
1425 checkpoint: Option<&[u8]>,
1426 validated: Option<&arachne_security::ValidatedAdmission>,
1427) -> Result<Value, String> {
1428 check_epoch_transition(session)?;
1429 let workspace = session
1430 .workspace
1431 .as_ref()
1432 .ok_or("session has no workspace")?;
1433 if workspace
1434 .retained_admission(authenticated_endpoint, request)
1435 .map_err(str::to_owned)?
1436 .is_some()
1437 {
1438 return Err("admission already retained; retrieve its existing response".into());
1439 }
1440 let prepared = match validated {
1441 Some(validated) => workspace
1442 .prepare_validated_admission(authenticated_endpoint, request, validated)
1443 .map_err(str::to_owned)?,
1444 None => workspace
1445 .prepare_admission(authenticated_endpoint, request)
1446 .map_err(str::to_owned)?,
1447 };
1448 if checkpoint.is_some() {
1450 admission_reply(
1451 &prepared.workspace,
1452 authenticated_endpoint,
1453 request,
1454 checkpoint,
1455 )?;
1456 }
1457 stage_admission_workspace(session, prepared.workspace, 1)
1458}
1459
1460fn stage_admission_workspace(
1461 session: &mut Session,
1462 workspace: arachne_security::Workspace,
1463 admission_count: usize,
1464) -> Result<Value, String> {
1465 let key = session
1466 .storage_key
1467 .as_ref()
1468 .ok_or("session has no protected root key")?;
1469 let snapshot = seal_state(session.records.is_some(), &workspace, key, None, None, None)?;
1470 let value = json!({"workspace": workspace.id(), "snapshot": snapshot, "state":"awaiting_save", "durable":false,
1471 "admissions": admission_count});
1472 session.staged_workspace = Some(StagedWorkspace {
1473 received: None,
1474 inbox: None,
1475 publisher: None, transition: WorkspaceTransition::Admission,
1477 workspace,
1478 snapshot,
1479 });
1480 Ok(value)
1481}
1482
1483fn retained_reply(
1484 workspace: &arachne_security::Workspace,
1485 authenticated_endpoint: [u8; 32],
1486 request: &[u8],
1487) -> Result<Value, String> {
1488 let reply = workspace
1489 .retained_admission(authenticated_endpoint, request)
1490 .map_err(str::to_owned)?
1491 .ok_or("no retained admission")?;
1492 Ok(
1493 json!({"workspace":workspace.id(), "epoch":reply.epoch, "commit":reply.commit, "welcome":reply.welcome,
1494 "authorization":{"invitation_key":reply.authorization.invitation_key,
1495 "grant_signature":reply.authorization.grant_signature.as_slice(),
1496 "redemption_signature":reply.authorization.redemption_signature.as_slice()}}),
1497 )
1498}
1499
1500type AdmissionPacket<'a> = (&'a [u8], Option<&'a [u8]>, Option<&'a str>);
1503
1504const ADMISSION_HISTORY_PAGE_REQUEST: &[u8; 5] = b"DFJP\x01";
1505
1506fn admission_packet(bytes: &[u8]) -> Result<AdmissionPacket<'_>, String> {
1507 if !bytes.starts_with(b"DFJA") {
1508 return Ok((bytes, None, None));
1509 }
1510 if bytes.len() < 9 || (!bytes.starts_with(b"DFJA\x01") && !bytes.starts_with(b"DFJA\x02")) {
1511 return Err("invalid admission packet".into());
1512 }
1513 let length = u32::from_be_bytes(bytes[5..9].try_into().unwrap()) as usize;
1514 let (header, name_length): (usize, usize) = if bytes.starts_with(b"DFJA\x02") {
1515 if bytes.len() < 11 {
1516 return Err("invalid admission packet bounds".into());
1517 }
1518 (
1519 11,
1520 u16::from_be_bytes(bytes[9..11].try_into().unwrap()) as usize,
1521 )
1522 } else {
1523 (9, 0)
1524 };
1525 let request_end = header
1526 .checked_add(length)
1527 .ok_or("invalid admission packet bounds")?;
1528 let name_end = request_end
1529 .checked_add(name_length)
1530 .ok_or("invalid admission packet bounds")?;
1531 if length == 0 || name_end >= bytes.len() {
1532 return Err("invalid admission packet bounds".into());
1533 }
1534 let name = if name_length == 0 {
1535 None
1536 } else {
1537 let value = std::str::from_utf8(&bytes[request_end..name_end])
1538 .map_err(|_| "invalid admission display name")?;
1539 if value.trim() != value
1540 || value.is_empty()
1541 || value.len() > 256
1542 || value.chars().count() > 80
1543 || value.chars().any(|c| {
1544 c.is_control()
1545 || matches!(c, '\u{061c}' | '\u{200e}' | '\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{2066}'..='\u{2069}')
1546 })
1547 {
1548 return Err("invalid admission display name".into());
1549 }
1550 Some(value)
1551 };
1552 Ok((&bytes[header..request_end], Some(&bytes[name_end..]), name))
1553}
1554
1555fn admission_parts(bytes: &[u8]) -> Result<(&[u8], Option<&[u8]>), String> {
1556 let (request, checkpoint, _) = admission_packet(bytes)?;
1557 Ok((request, checkpoint))
1558}
1559
1560fn admission_history_page_packet(
1561 request: &[u8],
1562 checkpoint: &[u8],
1563 offset: usize,
1564) -> Result<Vec<u8>, String> {
1565 let request_len = u32::try_from(request.len()).map_err(|_| "admission request too large")?;
1566 let checkpoint_len = u32::try_from(checkpoint.len()).map_err(|_| "checkpoint too large")?;
1567 let offset = u32::try_from(offset).map_err(|_| "history offset too large")?;
1568 let mut packet = ADMISSION_HISTORY_PAGE_REQUEST.to_vec();
1569 packet.extend(request_len.to_be_bytes());
1570 packet.extend(checkpoint_len.to_be_bytes());
1571 packet.extend(offset.to_be_bytes());
1572 packet.extend(request);
1573 packet.extend(checkpoint);
1574 Ok(packet)
1575}
1576
1577fn join_attempt_error(error: arachne_node::Error, initial: bool) -> JoinAttemptOutcome {
1578 match error {
1579 arachne_node::Error::ControlNotSent(_) | arachne_node::Error::MissingPeer if initial => {
1580 JoinAttemptOutcome::NotSent
1581 }
1582 arachne_node::Error::ControlNotSent(_)
1583 | arachne_node::Error::MissingPeer
1584 | arachne_node::Error::Timeout(_)
1585 | arachne_node::Error::Transport(_) => JoinAttemptOutcome::Waiting,
1586 error => JoinAttemptOutcome::Failed(error.to_string()),
1587 }
1588}
1589
1590async fn request_invitation_checkpoint(
1591 client: ControlClient,
1592 requester: [u8; 32],
1593 invitation: Vec<u8>,
1594 peers: Vec<[u8; 32]>,
1595) -> Result<Value, String> {
1596 let invitation =
1597 arachne_security::Invitation::from_bytes(&invitation).map_err(str::to_owned)?;
1598 if peers.is_empty()
1599 || peers.len() > 3
1600 || peers.iter().any(|peer| peer == &requester)
1601 || peers
1602 .iter()
1603 .enumerate()
1604 .any(|(index, peer)| peers[..index].contains(peer))
1605 {
1606 return Err("checkpoint discovery requires one to three distinct peers".into());
1607 }
1608 let mut last_error = None;
1609 for peer in peers {
1610 let mut packet = INVITATION_CHECKPOINT_REQUEST.to_vec();
1611 packet.extend(
1612 invitation
1613 .checkpoint_request(requester, peer)
1614 .map_err(str::to_owned)?,
1615 );
1616 let checkpoint = match client.clone().request_control(peer, &packet).await {
1617 Ok(checkpoint) => checkpoint,
1618 Err(error) => {
1619 last_error = Some(error.to_string());
1620 continue;
1621 }
1622 };
1623 match invitation.join_proof(&checkpoint) {
1624 Ok(_) => {
1625 return Ok(json!({
1626 "workspace": invitation.workspace_id(),
1627 "checkpoint": checkpoint,
1628 "peer": peer,
1629 }));
1630 }
1631 Err(error) => last_error = Some(error.to_string()),
1632 }
1633 }
1634 Err(last_error.unwrap_or_else(|| "no authorized workspace member is reachable".into()))
1635}
1636
1637async fn request_join_exchange(
1638 client: ControlClient,
1639 peer: [u8; 32],
1640 request: Vec<u8>,
1641 name: Vec<u8>,
1642 checkpoint: Vec<u8>,
1643) -> JoinAttemptOutcome {
1644 let mut packet = b"DFJA\x02".to_vec();
1645 let request_len = match u32::try_from(request.len()) {
1646 Ok(length) => length,
1647 Err(_) => return JoinAttemptOutcome::Failed("admission request too large".into()),
1648 };
1649 let name_len = match u16::try_from(name.len()) {
1650 Ok(length) => length,
1651 Err(_) => return JoinAttemptOutcome::Failed("admission display name too large".into()),
1652 };
1653 packet.extend(request_len.to_be_bytes());
1654 packet.extend(name_len.to_be_bytes());
1655 packet.extend(&request);
1656 packet.extend(&name);
1657 packet.extend(&checkpoint);
1658
1659 let first = match client.clone().request_control(peer, &packet).await {
1660 Ok(reply) => reply,
1661 Err(error) => return join_attempt_error(error, true),
1662 };
1663 let mut page_bytes = vec![first.len()];
1664 let mut reply: Value = match serde_json::from_slice(&first) {
1665 Ok(value) => value,
1666 Err(_) => return JoinAttemptOutcome::Failed("invalid admission reply".into()),
1667 };
1668 if reply
1669 .get("history_complete")
1670 .is_some_and(|complete| !complete.as_bool().unwrap_or(false))
1671 {
1672 let mut commits = match reply["commits"].as_array().cloned() {
1673 Some(commits) => commits,
1674 None => {
1675 return JoinAttemptOutcome::Failed("admission history page missing commits".into());
1676 }
1677 };
1678 let mut offset = match reply["history_next"].as_u64() {
1679 Some(offset) => offset as usize,
1680 None => {
1681 return JoinAttemptOutcome::Failed(
1682 "admission history page missing next offset".into(),
1683 );
1684 }
1685 };
1686 let mut page_count = 0;
1687 let mut total_bytes = first.len();
1688 while !reply["history_complete"].as_bool().unwrap_or(false) {
1689 page_count += 1;
1690 if page_count > arachne_security::MAX_JOIN_HISTORY_STEPS {
1691 return JoinAttemptOutcome::Failed(
1692 "admission history page count exceeds bounds".into(),
1693 );
1694 }
1695 let page = match admission_history_page_packet(&request, &checkpoint, offset) {
1696 Ok(packet) => packet,
1697 Err(error) => return JoinAttemptOutcome::Failed(error),
1698 };
1699 let page = match client.clone().request_control(peer, &page).await {
1700 Ok(page) => page,
1701 Err(error) => return join_attempt_error(error, false),
1702 };
1703 total_bytes = total_bytes.saturating_add(page.len());
1704 if total_bytes > arachne_security::MAX_JOIN_HISTORY_BYTES {
1705 return JoinAttemptOutcome::Failed(
1706 "admission history exceeds transport bounds".into(),
1707 );
1708 }
1709 page_bytes.push(page.len());
1710 let page: Value = match serde_json::from_slice(&page) {
1711 Ok(value) => value,
1712 Err(_) => {
1713 return JoinAttemptOutcome::Failed("invalid admission history page".into());
1714 }
1715 };
1716 if page.get("history_page").and_then(Value::as_bool) != Some(true) {
1717 return JoinAttemptOutcome::Failed(
1718 "admission history page was not accepted".into(),
1719 );
1720 }
1721 if page["history_offset"].as_u64() != Some(offset as u64) {
1722 return JoinAttemptOutcome::Failed("admission history page offset mismatch".into());
1723 }
1724 let page_commits = match page["commits"].as_array() {
1725 Some(commits) => commits,
1726 None => {
1727 return JoinAttemptOutcome::Failed(
1728 "admission history page missing commits".into(),
1729 );
1730 }
1731 };
1732 let next = match page["history_next"].as_u64() {
1733 Some(next) => next as usize,
1734 None => {
1735 return JoinAttemptOutcome::Failed(
1736 "admission history page missing next offset".into(),
1737 );
1738 }
1739 };
1740 if page_commits.is_empty() || next <= offset {
1741 return JoinAttemptOutcome::Failed(
1742 "admission history page made no progress".into(),
1743 );
1744 }
1745 if commits.len() + page_commits.len() > arachne_security::MAX_JOIN_HISTORY_STEPS {
1746 return JoinAttemptOutcome::Failed("admission history exceeds step bounds".into());
1747 }
1748 commits.extend(page_commits.iter().cloned());
1749 offset = next;
1750 reply = page;
1751 }
1752 reply["commits"] = Value::Array(commits);
1753 reply["history_complete"] = Value::Bool(true);
1754 }
1755
1756 let mut history_prefix = Vec::new();
1757 let total = reply["commits"].as_array().map_or(0, Vec::len);
1758 if total > arachne_security::HISTORY_CHUNK_STEPS {
1759 let trailing = match total % arachne_security::HISTORY_CHUNK_STEPS {
1760 0 => arachne_security::HISTORY_CHUNK_STEPS,
1761 remainder => remainder,
1762 };
1763 let split = total - trailing;
1764 let commits = reply["commits"].as_array().unwrap();
1765 history_prefix = commits[..split].to_vec();
1766 reply["commits"] = Value::Array(commits[split..].to_vec());
1767 reply["history_verified_prefix"] = json!(split);
1768 }
1769 if reply.get("commits").is_some() {
1770 reply["history_page_bytes"] = json!(page_bytes);
1771 }
1772 JoinAttemptOutcome::Reply {
1773 value: reply,
1774 history_prefix,
1775 }
1776}
1777
1778fn parse_admission_history_page_packet(bytes: &[u8]) -> Result<(&[u8], &[u8], usize), String> {
1779 if !bytes.starts_with(ADMISSION_HISTORY_PAGE_REQUEST) || bytes.len() < 17 {
1780 return Err("invalid admission history page request".into());
1781 }
1782 let request_len = u32::from_be_bytes(bytes[5..9].try_into().unwrap()) as usize;
1783 let checkpoint_len = u32::from_be_bytes(bytes[9..13].try_into().unwrap()) as usize;
1784 let offset = u32::from_be_bytes(bytes[13..17].try_into().unwrap()) as usize;
1785 let request_end = 17usize
1786 .checked_add(request_len)
1787 .ok_or("invalid admission history page bounds")?;
1788 let checkpoint_end = request_end
1789 .checked_add(checkpoint_len)
1790 .ok_or("invalid admission history page bounds")?;
1791 if request_len == 0 || checkpoint_end != bytes.len() {
1792 return Err("invalid admission history page bounds".into());
1793 }
1794 Ok((
1795 &bytes[17..request_end],
1796 &bytes[request_end..checkpoint_end],
1797 offset,
1798 ))
1799}
1800
1801const INVITATION_CHECKPOINT_REQUEST: &[u8; 5] = b"DFIC\x01";
1802
1803fn invitation_checkpoint_reply(
1804 session: &Session,
1805 requester: [u8; 32],
1806 request: &[u8],
1807) -> Result<Vec<u8>, String> {
1808 let proof = request
1809 .strip_prefix(INVITATION_CHECKPOINT_REQUEST)
1810 .ok_or("invalid invitation checkpoint request")?;
1811 let workspace = session
1812 .workspace
1813 .as_ref()
1814 .ok_or("session has no workspace")?;
1815 workspace
1816 .checkpoint_for_invitation(requester, session.node.id(), proof)
1817 .map_err(str::to_owned)
1818}
1819
1820#[test]
1821fn admission_packet_rejects_invalid_version_and_lengths() {
1822 let packet = [b"DFJA\x01".as_slice(), &1u32.to_be_bytes(), &[7, 8]].concat();
1823 assert_eq!(
1824 admission_parts(&packet).unwrap(),
1825 (&[7][..], Some(&[8][..]))
1826 );
1827 let named = [
1828 b"DFJA\x02".as_slice(),
1829 &1u32.to_be_bytes(),
1830 &4u16.to_be_bytes(),
1831 &[7],
1832 b"Alex",
1833 &[8],
1834 ]
1835 .concat();
1836 assert_eq!(
1837 admission_packet(&named).unwrap(),
1838 (&[7][..], Some(&[8][..]), Some("Alex"))
1839 );
1840 for length in 4..packet.len() {
1841 assert!(admission_parts(&packet[..length]).is_err());
1842 }
1843 for length in [0, 2, u32::MAX] {
1844 let mut bad = packet.clone();
1845 bad[5..9].copy_from_slice(&length.to_be_bytes());
1846 assert!(admission_parts(&bad).is_err());
1847 }
1848 let mut bad = packet;
1849 bad[4] = 2;
1850 assert!(admission_parts(&bad).is_err());
1851}
1852
1853fn admission_reply(
1854 workspace: &arachne_security::Workspace,
1855 peer: [u8; 32],
1856 request: &[u8],
1857 checkpoint: Option<&[u8]>,
1858) -> Result<Vec<u8>, String> {
1859 admission_reply_page(workspace, peer, request, checkpoint, 0)
1860}
1861
1862fn send_inbound_admission_reply(session: &mut Session) -> Result<Value, String> {
1863 let incoming = session
1864 .inbound_admission
1865 .take()
1866 .ok_or("session has no received admission")?;
1867 let workspace = session
1868 .workspace
1869 .as_ref()
1870 .ok_or("session has no workspace")?;
1871 let reply = if incoming.payload().starts_with(b"DFLV") {
1872 membership::leave_reply(workspace, incoming.peer(), incoming.payload())?
1873 } else if incoming.payload().starts_with(b"DFMO")
1874 || incoming.payload().starts_with(ADMISSION_RESULT_OFFER)
1875 {
1876 vec![1] } else {
1878 let (request, checkpoint) = admission_parts(incoming.payload())?;
1879 admission_reply(workspace, incoming.peer(), request, checkpoint)?
1880 };
1881 let queued = match incoming.respond(reply) {
1882 Ok(()) => true,
1883 Err(arachne_node::Error::Rejected) => false, Err(error) => return Err(error.to_string()),
1885 };
1886 Ok(json!({"queued":queued,"remote_receipt":false}))
1887}
1888
1889fn admission_reply_page(
1890 workspace: &arachne_security::Workspace,
1891 peer: [u8; 32],
1892 request: &[u8],
1893 checkpoint: Option<&[u8]>,
1894 offset: usize,
1895) -> Result<Vec<u8>, String> {
1896 let mut reply = retained_reply(workspace, peer, request)?;
1897 if let Some(checkpoint) = checkpoint {
1898 let mut steps = workspace
1899 .membership_history(peer, request, checkpoint)
1900 .map_err(str::to_owned)?;
1901 let last = steps
1902 .iter()
1903 .position(|(_, commit)| json!(commit) == reply["commit"])
1904 .ok_or("retained admission missing from verified history")?;
1905 steps.truncate(last + 1); if offset >= steps.len() {
1907 return Err("admission history page offset is out of bounds".into());
1908 }
1909 let mut page = Vec::new();
1910 let mut next = offset;
1911 for (authorization, commit) in steps.iter().skip(offset) {
1912 let step = membership::step_json(authorization, commit);
1913 let mut candidate = page.clone();
1914 candidate.push(step);
1915 reply["commits"] = json!(candidate);
1916 reply["history_offset"] = json!(offset);
1917 reply["history_next"] = json!(next + 1);
1918 reply["history_complete"] = json!(next + 1 == steps.len());
1919 if serde_json::to_vec(&reply).map_err(|e| e.to_string())?.len()
1920 > arachne_node::MAX_CONTROL_REPLY
1921 {
1922 if page.is_empty() {
1923 return Err("admission history step exceeds transport bound".into());
1924 }
1925 break;
1926 }
1927 page = candidate;
1928 next += 1;
1929 }
1930 reply["commits"] = json!(page);
1931 if next < steps.len() || offset > 0 {
1932 reply["history_offset"] = json!(offset);
1933 reply["history_next"] = json!(next);
1934 reply["history_complete"] = json!(next == steps.len());
1935 reply["history_page"] = json!(true);
1936 } else if let Some(object) = reply.as_object_mut() {
1937 object.remove("history_offset");
1938 object.remove("history_next");
1939 object.remove("history_complete");
1940 object.remove("history_page");
1941 }
1942 }
1943 let encoded = serde_json::to_vec(&reply).map_err(|e| e.to_string())?;
1944 if encoded.len() > arachne_node::MAX_CONTROL_REPLY {
1945 return Err("admission reply exceeds transport bound".into());
1946 }
1947 Ok(encoded)
1948}
1949
1950fn admission_packet_candidate(bytes: &[u8]) -> bool {
1951 bytes.starts_with(b"DFJA")
1952 || bytes.starts_with(b"DFJR\x01")
1953 || bytes.starts_with(ADMISSION_HISTORY_PAGE_REQUEST)
1954}
1955
1956fn admission_offer_candidate(bytes: &[u8]) -> bool {
1957 bytes.starts_with(ADMISSION_RESULT_OFFER)
1958}
1959
1960fn admission_offer_packet(reply: &[u8]) -> Result<Vec<u8>, String> {
1961 let length = u32::try_from(reply.len()).map_err(|_| "admission result is too large")?;
1962 if reply.len() > 32 * 1024 - 9 {
1963 return Err("admission result offer exceeds control bound".into());
1964 }
1965 let mut packet = ADMISSION_RESULT_OFFER.to_vec();
1966 packet.extend(length.to_be_bytes());
1967 packet.extend(reply);
1968 Ok(packet)
1969}
1970
1971fn parse_admission_offer(packet: &[u8]) -> Result<(Vec<JoinStep>, Vec<u8>), String> {
1972 if packet.len() < 9 || !packet.starts_with(ADMISSION_RESULT_OFFER) {
1973 return Err("invalid admission result offer".into());
1974 }
1975 let length = u32::from_be_bytes(packet[5..9].try_into().unwrap()) as usize;
1976 if length == 0 || packet.len() != 9 + length || packet.len() > 32 * 1024 {
1977 return Err("invalid admission result offer bounds".into());
1978 }
1979 let reply: Value = serde_json::from_slice(&packet[9..]).map_err(|_| "invalid admission result")?;
1980 if reply.get("history_complete").and_then(Value::as_bool) == Some(false) {
1981 return Err("paged admission result requires the retry path".into());
1982 }
1983 let commits = if let Some(commits) = reply.get("commits") {
1984 serde_json::from_value(commits.clone()).map_err(|_| "invalid admission result commits")?
1985 } else {
1986 vec![serde_json::from_value(json!({
1987 "commit": reply["commit"].clone(),
1988 "authorization": reply["authorization"].clone(),
1989 }))
1990 .map_err(|_| "incomplete admission result")?]
1991 };
1992 if commits.is_empty() || commits.len() > arachne_security::HISTORY_CHUNK_STEPS {
1993 return Err("admission result history exceeds bounds".into());
1994 }
1995 let welcome = serde_json::from_value(reply["welcome"].clone())
1996 .map_err(|_| "admission result has no Welcome")?;
1997 Ok((commits, welcome))
1998}
1999
2000fn reap_admission_pushes(session: &mut Session) {
2001 session
2002 .admission_pushes
2003 .retain(|push| !push.task.is_finished());
2004}
2005
2006fn queue_admission_push(
2007 session: &mut Session,
2008 attempt: &arachne_security::AdmissionAttempt,
2009 reply: &[u8],
2010 route: Option<std::net::SocketAddr>,
2011) -> bool {
2012 if serde_json::from_slice::<Value>(reply)
2013 .ok()
2014 .and_then(|value| value.get("history_complete").and_then(Value::as_bool))
2015 == Some(false)
2016 {
2017 return false;
2018 }
2019 let Ok(packet) = admission_offer_packet(reply) else {
2020 return false;
2021 };
2022 let peer = attempt.endpoint();
2023 if let Some(route) = route {
2024 let _ = session
2025 .runtime
2026 .block_on(session.node.add_address_hint(peer, route));
2027 }
2028 let client = session.node.control_client();
2033 let wake = session.node.control_signal();
2034 let task = session.runtime.spawn(async move {
2035 let result = client.request_control(peer, &packet).await;
2036 wake.notify_one();
2037 result
2038 });
2039 session.admission_pushes.push(PendingControl {
2040 query: attempt.id(),
2041 peer,
2042 task,
2043 });
2044 true
2045}
2046
2047fn admission_reason(error: &str) -> &'static str {
2048 match error {
2049 arachne_security::INVITATION_AUTOMATIC_APPROVAL_REQUIRED => "automatic_approval_required",
2050 arachne_security::INVITATION_APPROVAL_REQUIRED => "approval_required",
2051 arachne_security::INVITATION_DISABLED => "invitation_disabled",
2052 arachne_security::INVITATION_EXPIRED => "invitation_expired",
2053 "member already admitted" => admission_state::MEMBER_ALREADY_ADMITTED,
2054 _ => "unavailable",
2055 }
2056}
2057
2058fn admission_feedback(error: &str) -> Value {
2059 let reason = admission_reason(error);
2060 if reason == admission_state::MEMBER_ALREADY_ADMITTED {
2061 json!({
2062 "state": admission_state::RECOVERY_REQUIRED,
2063 "reason": reason,
2064 "recovery": admission_state::RECOVERY_REMOVE_AND_REINVITE,
2065 })
2066 } else {
2067 json!({"state": admission_state::UNAVAILABLE, "reason": reason})
2068 }
2069}
2070
2071fn enqueue_admission(
2072 session: &mut Session,
2073 attempt: arachne_security::AdmissionAttempt,
2074 checkpoint: Option<Vec<u8>>,
2075 display_name: Option<String>,
2076 approval_automatic: Option<bool>,
2077 validated: arachne_security::ValidatedAdmission,
2078) -> Result<arachne_security::AdmissionEnqueue, arachne_security::AdmissionQueueError> {
2079 let id = attempt.id();
2080 let result = session.admission_queue.enqueue(attempt)?;
2081 if result == arachne_security::AdmissionEnqueue::Added {
2082 session.admission_metadata.insert(
2083 id,
2084 QueuedAdmission {
2085 checkpoint,
2086 display_name,
2087 approval_automatic,
2088 validated,
2089 },
2090 );
2091 }
2092 Ok(result)
2093}
2094
2095fn hold_admission_exchange(
2098 session: &mut Session,
2099 attempt: [u8; 32],
2100 incoming: arachne_node::ControlRequest,
2101 checkpoint: Option<Vec<u8>>,
2102) {
2103 if let Some(overflow) = session
2104 .admission_waiters
2105 .hold(attempt, incoming, checkpoint)
2106 {
2107 let _ = overflow.respond(b"{\"state\":\"admission_queued\"}".to_vec());
2108 }
2109}
2110
2111fn queue_admission(
2112 session: &mut Session,
2113 incoming: arachne_node::ControlRequest,
2114) -> Result<Value, String> {
2115 session.admission_reads_since_stage = session.admission_reads_since_stage.saturating_add(1);
2116 let peer = incoming.peer();
2117 let packet = incoming.payload().to_vec();
2118 if packet.starts_with(ADMISSION_HISTORY_PAGE_REQUEST) {
2119 let response = match parse_admission_history_page_packet(&packet) {
2120 Ok((request, checkpoint, offset)) => session.workspace.as_ref().map_or_else(
2121 || b"{\"state\":\"admission_unavailable\",\"reason\":\"unavailable\"}".to_vec(),
2122 |workspace| {
2123 admission_reply_page(workspace, peer, request, Some(checkpoint), offset)
2124 .unwrap_or_else(|_| {
2125 b"{\"state\":\"admission_unavailable\",\"reason\":\"unavailable\"}"
2126 .to_vec()
2127 })
2128 },
2129 ),
2130 Err(_) => b"{\"state\":\"admission_unavailable\",\"reason\":\"unavailable\"}".to_vec(),
2131 };
2132 let accepted = incoming.respond(response).is_ok();
2133 return Ok(json!({"state":"admission_replied", "accepted":accepted,
2134 "history_page":true}));
2135 }
2136 let (request, checkpoint, display_name) = match admission_packet(&packet) {
2137 Ok((request, checkpoint, display_name)) => (
2138 request.to_vec(),
2139 checkpoint.map(ToOwned::to_owned),
2140 display_name.map(str::to_owned),
2141 ),
2142 Err(_) => {
2143 let _ = incoming.respond(
2144 serde_json::to_vec(
2145 &json!({"state":"admission_unavailable","reason":"unavailable"}),
2146 )
2147 .map_err(|_| "admission feedback encoding failed")?,
2148 );
2149 return Ok(json!({"state":"admission_replied","accepted":false}));
2150 }
2151 };
2152 let attempt =
2153 arachne_security::AdmissionAttempt::new(peer, request.clone()).map_err(str::to_owned)?;
2154 {
2155 let workspace = session
2156 .workspace
2157 .as_ref()
2158 .ok_or("session has no workspace")?;
2159 if workspace
2160 .retained_admission(peer, &request)
2161 .map_err(str::to_owned)?
2162 .is_some()
2163 {
2164 let reply = admission_reply_page(workspace, peer, &request, checkpoint.as_deref(), 0)?;
2165 let accepted = incoming.respond(reply).is_ok();
2166 if accepted {
2167 session
2168 .queued_admission_in_flight
2169 .retain(|queued| queued != &attempt);
2170 }
2171 return Ok(json!({"state":"admission_replied", "accepted":accepted}));
2172 }
2173 }
2174 if session.admission_queue.contains(&attempt)
2175 || session
2176 .queued_admission_in_flight
2177 .iter()
2178 .any(|queued| queued == &attempt)
2179 {
2180 hold_admission_exchange(session, attempt.id(), incoming, checkpoint);
2181 return Ok(json!({"state":"admission_queued"}));
2182 }
2183
2184 let assessment = session
2185 .workspace
2186 .as_ref()
2187 .ok_or("session has no workspace")?
2188 .assess_admission(peer, &request);
2189 if session.pending_approvals.contains_key(&attempt.id()) {
2190 let still_needs_approval = matches!(
2191 &assessment,
2192 Ok(arachne_security::AdmissionAssessment::ApprovalRequired(_))
2193 | Ok(arachne_security::AdmissionAssessment::AutomaticApprovalRequired(_))
2194 );
2195 if still_needs_approval {
2196 let _ = incoming.respond(b"{\"state\":\"admission_queued\"}".to_vec());
2197 return Ok(json!({"state":"admission_queued"}));
2198 }
2199 session.pending_approvals.remove(&attempt.id());
2202 }
2203 let (validated, approval_automatic) = match assessment {
2204 Ok(arachne_security::AdmissionAssessment::Ready(request)) => (request, None),
2205 Ok(arachne_security::AdmissionAssessment::ApprovalRequired(request)) => {
2206 (request, Some(false))
2207 }
2208 Ok(arachne_security::AdmissionAssessment::AutomaticApprovalRequired(request)) => {
2209 (request, Some(true))
2210 }
2211 Err(error) => {
2212 let feedback = admission_feedback(error);
2213 let _ = incoming.respond(
2214 serde_json::to_vec(&feedback)
2215 .map_err(|_| "admission feedback encoding failed")?,
2216 );
2217 let mut result = json!({"state":admission_state::REPLIED,"accepted":false});
2218 for key in ["reason", "recovery"] {
2219 if let Some(value) = feedback.get(key) {
2220 result[key] = value.clone();
2221 }
2222 }
2223 if let Some(name) = display_name {
2224 result["display_name"] = json!(name);
2225 }
2226 return Ok(result);
2227 }
2228 };
2229
2230 if approval_automatic.is_some() {
2231 if enqueue_admission(
2232 session,
2233 attempt,
2234 checkpoint,
2235 display_name,
2236 approval_automatic,
2237 validated,
2238 )
2239 .is_err()
2240 {
2241 let _ = incoming.respond(
2242 b"{\"state\":\"admission_unavailable\",\"reason\":\"server_busy\"}".to_vec(),
2243 );
2244 return Ok(json!({"state":"admission_replied","accepted":false}));
2245 }
2246 let _ = incoming.respond(b"{\"state\":\"admission_queued\"}".to_vec());
2247 return Ok(json!({"state":"admission_queued"}));
2248 }
2249
2250 if let Some(checkpoint) = checkpoint.as_deref()
2251 && session
2252 .workspace
2253 .as_ref()
2254 .ok_or("session has no workspace")?
2255 .check_membership_history(peer, &request, checkpoint)
2259 .is_err()
2260 {
2261 let _ = incoming.respond(
2262 serde_json::to_vec(&json!({"state":"admission_unavailable","reason":"unavailable"}))
2263 .map_err(|_| "admission feedback encoding failed")?,
2264 );
2265 return Ok(json!({"state":"admission_replied","accepted":false}));
2266 }
2267 let attempt_id = attempt.id();
2268 let held_checkpoint = checkpoint.clone();
2269 if enqueue_admission(session, attempt, checkpoint, display_name, None, validated).is_err() {
2270 let _ = incoming
2271 .respond(b"{\"state\":\"admission_unavailable\",\"reason\":\"server_busy\"}".to_vec());
2272 return Ok(json!({"state":"admission_replied","accepted":false}));
2273 }
2274 hold_admission_exchange(session, attempt_id, incoming, held_checkpoint);
2275 Ok(json!({"state":"admission_queued","intake":true}))
2279}
2280
2281fn should_stage_queued_admission(session: &Session) -> bool {
2287 !session.admission_queue.is_empty()
2288 && (session.admission_queue.len() >= MAX_RUNTIME_ADMISSION_BATCH
2289 || session.admission_reads_since_stage >= MAX_RUNTIME_ADMISSION_BATCH)
2290}
2291
2292fn stage_queued_admission(session: &mut Session) -> Result<Option<Value>, String> {
2293 session.admission_reads_since_stage = 0;
2294 let mut ready = Vec::new();
2295 let mut deferred = Vec::new();
2296 while ready.len() < MAX_RUNTIME_ADMISSION_BATCH {
2297 let Some(attempt) = session.admission_queue.pop() else {
2298 break;
2299 };
2300 let Some(queued) = session.admission_metadata.remove(&attempt.id()) else {
2301 continue;
2302 };
2303 if queued.approval_automatic.is_some() {
2304 deferred.push((attempt, queued));
2305 continue;
2306 }
2307 if session
2310 .workspace
2311 .as_ref()
2312 .and_then(|workspace| {
2313 workspace
2314 .retained_admission(attempt.endpoint(), attempt.request())
2315 .ok()
2316 .flatten()
2317 })
2318 .is_some()
2319 {
2320 continue;
2321 }
2322 ready.push((attempt, queued));
2323 }
2324
2325 if ready.is_empty() {
2326 let mut deferred = deferred.into_iter();
2327 let Some((attempt, queued)) = deferred.next() else {
2328 return Ok(None);
2329 };
2330 for (attempt, queued) in deferred {
2331 requeue_admission(session, attempt, queued)?;
2332 }
2333 let id = attempt.id();
2334 let pending = session
2335 .pending_approvals
2336 .entry(id)
2337 .or_insert(PendingAdmissionApproval {
2338 attempt,
2339 queued,
2340 delivered: false,
2341 acknowledged: false,
2342 });
2343 pending.delivered = true;
2344 return Ok(Some(
2345 json!({"state":admission_state::APPROVAL_REQUESTED,"attempt_id":id,
2346 "endpoint":pending.attempt.endpoint(),"request":pending.attempt.request(),
2347 "display_name":pending.queued.display_name,
2348 "automatic":pending.queued.approval_automatic.unwrap_or(false)}),
2349 ));
2350 }
2351
2352 if let Err(error) = check_epoch_transition(session) {
2353 for (attempt, queued) in deferred {
2354 requeue_admission(session, attempt, queued)?;
2355 }
2356 for (attempt, queued) in ready {
2357 requeue_admission(session, attempt, queued)?;
2358 }
2359 return Err(error);
2360 }
2361
2362 for (attempt, queued) in deferred {
2363 requeue_admission(session, attempt, queued)?;
2364 }
2365 let inputs: Vec<_> = ready
2366 .iter()
2367 .map(|(attempt, queued)| (attempt.endpoint(), attempt.request(), &queued.validated))
2368 .collect();
2369 let prepared = match session
2370 .workspace
2371 .as_ref()
2372 .ok_or("session has no workspace")?
2373 .prepare_validated_admission_batch(&inputs)
2374 {
2375 Ok(prepared) => prepared,
2376 Err(error) => {
2377 for (attempt, queued) in ready {
2378 requeue_admission(session, attempt, queued)?;
2379 }
2380 return Err(error.to_owned());
2381 }
2382 };
2383 let history_check = ready.iter().try_for_each(|(attempt, queued)| {
2384 queued.checkpoint.as_deref().map_or(Ok(()), |checkpoint| {
2385 admission_reply(
2386 &prepared.workspace,
2387 attempt.endpoint(),
2388 attempt.request(),
2389 Some(checkpoint),
2390 )
2391 .map(|_| ())
2392 })
2393 });
2394 if let Err(error) = history_check {
2395 for (attempt, queued) in ready {
2396 requeue_admission(session, attempt, queued)?;
2397 }
2398 return Err(error);
2399 }
2400 let count = ready.len();
2401 let mut value = match stage_admission_workspace(session, prepared.workspace, count) {
2402 Ok(value) => value,
2403 Err(error) => {
2404 for (attempt, queued) in ready {
2405 requeue_admission(session, attempt, queued)?;
2406 }
2407 return Err(error);
2408 }
2409 };
2410 let attempts = ready.into_iter().map(|(attempt, _)| attempt).collect();
2411 session.queued_admission_in_flight = attempts;
2412 value["queued"] = json!(true);
2413 Ok(Some(value))
2414}
2415
2416fn requeue_admission(
2417 session: &mut Session,
2418 attempt: arachne_security::AdmissionAttempt,
2419 queued: QueuedAdmission,
2420) -> Result<(), String> {
2421 let id = attempt.id();
2422 session
2423 .admission_queue
2424 .enqueue(attempt)
2425 .map_err(|_| "admission queue is full".to_owned())?;
2426 session.admission_metadata.insert(id, queued);
2427 Ok(())
2428}
2429
2430fn pending_approval_id(
2431 session: &Session,
2432 request: &[u8],
2433 requested: Option<[u8; 32]>,
2434) -> Result<Option<[u8; 32]>, String> {
2435 if let Some(id) = requested {
2436 let pending = session
2437 .pending_approvals
2438 .get(&id)
2439 .ok_or("admission approval is no longer pending")?;
2440 if pending.attempt.request() != request {
2441 return Err("admission approval request does not match attempt".into());
2442 }
2443 return Ok(Some(id));
2444 }
2445 Ok(session
2446 .pending_approvals
2447 .iter()
2448 .find(|(_, pending)| pending.attempt.request() == request)
2449 .map(|(id, _)| *id))
2450}
2451
2452fn list_pending_approvals(
2453 session: &Session,
2454 after: Option<[u8; 32]>,
2455 limit: Option<usize>,
2456) -> Result<Value, String> {
2457 let limit = limit.unwrap_or(64);
2458 if !(1..=64).contains(&limit) {
2459 return Err("approval page limit must be between 1 and 64".into());
2460 }
2461 let rows: Vec<_> = session
2462 .pending_approvals
2463 .iter()
2464 .filter(|(id, _)| after.is_none_or(|after| **id > after))
2465 .take(limit + 1)
2466 .collect();
2467 let complete = rows.len() <= limit;
2468 let rows = rows.into_iter().take(limit).map(|(id, pending)| {
2469 json!({
2470 "attempt_id": id,
2471 "endpoint": pending.attempt.endpoint(),
2472 "request": pending.attempt.request(),
2473 "display_name": pending.queued.display_name,
2474 "automatic": pending.queued.approval_automatic.unwrap_or(false),
2475 "delivered": pending.delivered,
2476 "acknowledged": pending.acknowledged,
2477 })
2478 });
2479 let approvals: Vec<_> = rows.collect();
2480 let next_after = approvals
2481 .last()
2482 .and_then(|value| value["attempt_id"].as_array())
2483 .and_then(|bytes| bytes.iter().map(Value::as_u64).collect::<Option<Vec<_>>>())
2484 .and_then(|bytes| {
2485 (bytes.len() == 32 && bytes.iter().all(|byte| *byte <= u8::MAX as u64))
2486 .then(|| bytes.into_iter().map(|byte| byte as u8).collect::<Vec<_>>())
2487 });
2488 Ok(json!({
2489 "state": admission_state::APPROVAL_PENDING,
2490 "approvals": approvals,
2491 "complete": complete,
2492 "next_after": next_after,
2493 }))
2494}
2495
2496pub fn execute(handle: i64, bytes: &[u8]) -> Result<Vec<u8>, String> {
2498 if bytes.len() > MAX_REQUEST {
2499 return Err("request exceeds limit".into());
2500 }
2501 let request: Request = serde_json::from_slice(bytes).map_err(|e| e.to_string())?;
2502 serde_json::to_vec(&execute_request(handle, request)?).map_err(|e| e.to_string())
2503}
2504
2505pub fn execute_stored(
2510 handle: i64,
2511 metadata: &[u8],
2512 snapshot: &[u8],
2513) -> Result<[Vec<u8>; 2], String> {
2514 if metadata.len() > MAX_REQUEST || snapshot.len() > arachne_security::MAX_SEALED_BUNDLE {
2515 return Err("stored request exceeds limit".into());
2516 }
2517 let mut request: Request = serde_json::from_slice(metadata).map_err(|e| e.to_string())?;
2519 let value: Value = serde_json::from_slice(metadata).map_err(|e| e.to_string())?;
2520 let object = value.as_object().ok_or("request must be an object")?;
2521 if object.contains_key("snapshot") {
2522 return Err("snapshot must use the binary argument".into());
2523 }
2524 let stored = matches!(
2525 object.get("op").and_then(Value::as_str),
2526 Some(
2527 "restore_workspace"
2528 | "restore_pending_join"
2529 | "adopt_admission"
2530 | "adopt_join"
2531 | "adopt_publication"
2532 | "adopt_reception"
2533 | "adopt_recovery"
2534 | "adopt_current_view"
2535 )
2536 );
2537 let binary_welcome = matches!(request, Request::StageJoin { .. }) && !snapshot.is_empty();
2538 if binary_welcome {
2539 if object.contains_key("welcome") {
2540 return Err("Welcome must have exactly one representation".into());
2541 }
2542 if snapshot.len() > arachne_security::MAX_WELCOME {
2543 return Err("Welcome exceeds binary input bound".into());
2544 }
2545 if let Request::StageJoin { welcome, .. } = &mut request {
2546 *welcome = snapshot.to_vec();
2547 }
2548 }
2549 if !stored && !binary_welcome && !snapshot.is_empty() {
2550 return Err("operation does not accept a snapshot".into());
2551 }
2552 if stored {
2553 let target = match &mut request {
2554 Request::RestoreWorkspace { snapshot, .. }
2555 | Request::RestorePendingJoin { snapshot, .. }
2556 | Request::AdoptAdmission { snapshot }
2557 | Request::AdoptJoin { snapshot }
2558 | Request::AdoptPublication { snapshot }
2559 | Request::AdoptReception { snapshot }
2560 | Request::AdoptRecovery { snapshot }
2561 | Request::AdoptCurrentView { snapshot } => snapshot,
2562 _ => unreachable!(),
2563 };
2564 *target = snapshot.to_vec();
2565 }
2566 let mut response = execute_request(handle, request)?;
2567 let snapshot = response
2570 .as_object_mut()
2571 .and_then(|v| v.remove("snapshot"))
2572 .map(serde_json::from_value::<Vec<u8>>)
2573 .transpose()
2574 .map_err(|e| e.to_string())?
2575 .unwrap_or_default();
2576 if snapshot.len() > arachne_security::MAX_SEALED_BUNDLE {
2577 return Err("stored response exceeds limit; close and restore".into());
2578 }
2579 Ok([
2580 serde_json::to_vec(&response).map_err(|e| e.to_string())?,
2581 snapshot,
2582 ])
2583}
2584
2585fn execute_request(handle: i64, request: Request) -> Result<Value, String> {
2586 let shared = session(handle)?;
2587 let mut guard = shared.lock().map_err(|_| "node session unavailable")?;
2588 let busy_before = admission_busy(guard.as_ref().ok_or("node is closed")?);
2589 let mut ended = None;
2590 let result = execute_in_session(&mut guard, request, &mut ended);
2591 if let Some(session) = guard.as_mut()
2595 && session.staged_workspace.is_none()
2596 {
2597 session.staged_step_received = false;
2598 }
2599 if let Some(session) = guard.as_ref()
2600 && session.staged_workspace.is_none()
2601 && session.inbound_admission.is_none()
2602 && !session.admission_queue.is_empty()
2603 {
2604 session.node.rearm_control_signal();
2607 }
2608 if let Some(session) = guard.as_ref()
2609 && busy_before
2610 && !admission_busy(session)
2611 && session.node.has_deferred_controls()
2612 {
2613 session.node.rearm_control_signal();
2614 }
2615 if let Some(session) = guard.as_ref()
2618 && !admission_busy(session)
2619 && session
2620 .workspace
2621 .as_ref()
2622 .is_some_and(|owner| session.gossip_steps_ahead.contains_key(&owner.epoch()))
2623 {
2624 session.node.rearm_control_signal();
2625 }
2626 drop(guard);
2629 if let Some(ended) = ended {
2630 shutdown_session(ended)?;
2631 }
2632 result
2633}
2634
2635fn admission_busy(session: &Session) -> bool {
2636 session.staged_workspace.is_some() || session.inbound_admission.is_some()
2637}
2638
2639fn reset_workspace(session: &mut Session) -> Result<Value, String> {
2640 let changed = session.workspace.is_some()
2641 || session.pending_join.is_some()
2642 || session.records.is_some()
2643 || session.activity.phase != WorkspacePhase::Empty;
2644 let mut resetting = session.activity.clone();
2645 resetting.transition(WorkspacePhase::Resetting, Some("reset_requested"))?;
2646 let reset_presence = presence::Presence::new()?;
2647 let durable = session.records.is_some();
2648 if durable {
2649 persistence::reset_records(session, &resetting)?;
2650 }
2651
2652 release_overlay_paths(&DEVICE_OVERLAY_PATHS, session.overlay_paths);
2653 session.overlay_paths = 0;
2654 session.presence = reset_presence;
2655 session.resources = resources::Jobs::default();
2656 session.interests.cancel();
2657 session.membership_update = None;
2658 session.membership_offer = None;
2659 session.membership_offer_requires_adoption = false;
2660 session.membership_peer_failures.clear();
2661 session.staged_step_received = false;
2662 session.gossip_steps_ahead.clear();
2663 session.membership_head = None;
2664 session.range_pull = None;
2665 session.gossip_counts = Arc::default();
2666 session.profile_pull = None;
2667 session.profiles_walked.clear();
2668 session.gossip_profiles_pending.clear();
2669 session.cutoff = None;
2670 session.current_view = None;
2671 session.ready_current_view = None;
2672 session.range = None;
2673 session.ready_range = None;
2674 session.direct_range = None;
2675 session.ready_direct_range = None;
2676 session.direct_miss = None;
2677 session.recovered.clear();
2678 session.publisher = None;
2679 session.received = None;
2680 session.inbox = None;
2681 session.inbound_admission = None;
2682 session.admission_queue = arachne_security::AdmissionQueue::new();
2683 session.admission_metadata.clear();
2684 session.admission_waiters = admission_waiters::AdmissionWaiters::new(MAX_ADMISSION_WAITERS);
2685 session.admission_pushes.clear();
2686 session.pending_approvals.clear();
2687 session.staged_approval_id = None;
2688 session.queued_admission_in_flight.clear();
2689 session.admission_reads_since_stage = 0;
2690 session.nearby_workspaces.clear();
2691 session.staged_workspace = None;
2692 session.peer_profile_summaries.clear();
2693 session.staged_removal = None;
2694 session.workspace = None;
2695 session.committed.clear();
2696 session.checkpoint_exchange = None;
2697 session.join_exchange = None;
2698 session.pending_join = None;
2699 session.join_lifecycle = None;
2700 session.join_history_prefix.clear();
2701 session.records = None;
2702 session.activity = WorkspaceActivity::default();
2703
2704 Ok(json!({
2705 "state": "reset",
2706 "changed": changed,
2707 "durable": durable,
2708 "reset_activity": resetting.projection(),
2709 "activity": activity_value(session),
2710 }))
2711}
2712
2713fn discard_workspace_candidate(session: &mut Session) -> Result<Value, String> {
2714 if session.staged_removal.is_some() {
2715 return Err("removed membership awaits durable adoption".into());
2716 }
2717 if session.inbound_admission.is_some() {
2718 return Err("received admission awaits adoption or reply".into());
2719 }
2720 let discarded = session.staged_workspace.take().is_some();
2721 let offer_cancelled = session.membership_offer.take().is_some();
2722 session.membership_offer_requires_adoption = false;
2723 session.staged_step_received = false;
2724 Ok(json!({
2725 "state": "workspace_candidate_discarded",
2726 "discarded": discarded,
2727 "offer_cancelled": offer_cancelled,
2728 }))
2729}
2730
2731fn execute_in_session(
2732 guard: &mut std::sync::MutexGuard<'_, Option<Session>>,
2733 request: Request,
2734 ended: &mut Option<Session>,
2735) -> Result<Value, String> {
2736 if matches!(request, Request::ResetWorkspace {}) {
2737 return reset_workspace(guard.as_mut().ok_or("node is closed")?);
2738 }
2739 if matches!(request, Request::DiscardWorkspaceCandidate {}) {
2740 return discard_workspace_candidate(guard.as_mut().ok_or("node is closed")?);
2741 }
2742 if matches!(request, Request::DriveWorkspace {}) {
2743 if guard.as_ref().ok_or("node is closed")?.records.is_none() {
2744 return Err("workspace lifecycle requires native record storage".into());
2745 }
2746 let mut staged =
2747 execute_in_session(guard, Request::PollAdmission { profile: false }, ended)?;
2748 let staged_state = staged
2749 .get("state")
2750 .and_then(Value::as_str)
2751 .map(str::to_owned);
2752 if staged_state.as_deref() != Some("awaiting_save") {
2753 if staged_state.as_deref() == Some("reply_ready")
2754 && guard
2755 .as_ref()
2756 .ok_or("node is closed")?
2757 .inbound_admission
2758 .is_some()
2759 {
2760 let reply = execute_in_session(guard, Request::SendAdmissionReply {}, ended)?;
2761 staged["state"] = json!("workspace_reply_ready");
2762 staged["reply_queued"] = reply["queued"].clone();
2763 }
2764 if staged_state.is_some() {
2770 staged["activity"] = activity_value(guard.as_ref().ok_or("node is closed")?);
2771 return Ok(staged);
2772 }
2773 let membership = execute_in_session(guard, Request::PollMembershipUpdate {}, ended)?;
2774 if !membership.is_null() {
2775 let state = membership
2776 .get("state")
2777 .and_then(Value::as_str)
2778 .map(str::to_owned);
2779 if matches!(
2780 state.as_deref(),
2781 Some("workspace_name_update_available" | "workspace_name_checkpoint_available")
2782 ) {
2783 let request = if state.as_deref() == Some("workspace_name_update_available") {
2784 Request::StageWorkspaceNameUpdate {
2785 name_record: serde_json::from_value(membership["name_record"].clone())
2786 .map_err(|_| "workspace name record is invalid")?,
2787 }
2788 } else {
2789 Request::StageWorkspaceNameCheckpoint {
2790 name_checkpoint: serde_json::from_value(
2791 membership["name_checkpoint"].clone(),
2792 )
2793 .map_err(|_| "workspace name checkpoint is invalid")?,
2794 }
2795 };
2796 let staged_name = execute_in_session(guard, request, ended)?;
2797 let snapshot: Vec<u8> = serde_json::from_value(staged_name["snapshot"].clone())
2798 .map_err(|_| "workspace name snapshot is invalid")?;
2799 {
2800 let session = guard.as_mut().ok_or("node is closed")?;
2801 persistence::commit_candidate(session, &snapshot)?;
2802 }
2803 let mut committed =
2804 execute_in_session(guard, Request::AdoptAdmission { snapshot }, ended)?;
2805 committed["state"] = json!("workspace_name_committed");
2806 committed["activity"] = activity_value(guard.as_ref().ok_or("node is closed")?);
2807 return Ok(committed);
2808 }
2809 let mut membership = membership;
2810 if matches!(
2811 state.as_deref(),
2812 Some(
2813 "membership_update_available"
2814 | "membership_current"
2815 | "membership_unavailable"
2816 | "membership_peer_behind"
2817 | "workspace_name_peer_behind"
2818 | "membership_update_stale"
2819 )
2820 ) {
2821 membership["membership_state"] = membership["state"].clone();
2822 membership["state"] = json!("membership_replied");
2823 }
2824 membership["activity"] = activity_value(guard.as_ref().ok_or("node is closed")?);
2825 return Ok(membership);
2826 }
2827 staged["activity"] = activity_value(guard.as_ref().ok_or("node is closed")?);
2828 return Ok(staged);
2829 }
2830 let snapshot: Vec<u8> = serde_json::from_value(staged["snapshot"].clone())
2831 .map_err(|_| "workspace candidate snapshot is invalid")?;
2832 {
2833 let session = guard.as_mut().ok_or("node is closed")?;
2834 persistence::commit_candidate(session, &snapshot)?;
2835 }
2836 let mut committed = execute_in_session(guard, Request::AdoptAdmission { snapshot }, ended)?;
2837 if guard
2838 .as_ref()
2839 .ok_or("node is closed")?
2840 .inbound_admission
2841 .is_some()
2842 {
2843 let reply = execute_in_session(guard, Request::SendAdmissionReply {}, ended)?;
2844 committed["reply_queued"] = reply["queued"].clone();
2845 }
2846 committed["state"] = json!("workspace_committed");
2847 committed["activity"] = activity_value(guard.as_ref().ok_or("node is closed")?);
2848 return Ok(committed);
2849 }
2850 if matches!(request, Request::DriveJoin {}) {
2851 if guard.as_ref().ok_or("node is closed")?.records.is_none() {
2852 return Err("join lifecycle requires native record storage".into());
2853 }
2854 if let Some(incoming) = guard
2855 .as_mut()
2856 .ok_or("node is closed")?
2857 .node
2858 .poll_control_matching(admission_offer_candidate)
2859 {
2860 let peer = incoming.peer();
2861 let allowed = guard
2862 .as_ref()
2863 .ok_or("node is closed")?
2864 .join_lifecycle
2865 .as_ref()
2866 .is_some_and(|lifecycle| {
2867 lifecycle.peers.contains(&peer)
2868 && lifecycle.selected.is_none_or(|selected| selected == peer)
2869 });
2870 if !allowed {
2871 let _ = incoming.respond(vec![0]);
2872 return Ok(json!({"state":admission_state::UNAVAILABLE,
2873 "reason":"unrecognized_admission_pusher","peer":peer}));
2874 }
2875 if let Some(exchange) = guard
2876 .as_mut()
2877 .ok_or("node is closed")?
2878 .join_exchange
2879 .take()
2880 {
2881 exchange.task.abort();
2882 }
2883 let packet = incoming.payload().to_vec();
2884 let (commits, welcome) = match parse_admission_offer(&packet) {
2885 Ok(value) => value,
2886 Err(_) => {
2887 let _ = incoming.respond(vec![0]);
2888 return Ok(json!({"state":admission_state::UNAVAILABLE,
2889 "reason":"invalid_admission_offer","peer":peer}));
2890 }
2891 };
2892 let staged = match execute_in_session(
2893 guard,
2894 Request::StageJoin { commits, welcome },
2895 ended,
2896 ) {
2897 Ok(staged) => staged,
2898 Err(_) => {
2899 let _ = incoming.respond(vec![0]);
2900 return Ok(json!({"state":admission_state::UNAVAILABLE,
2901 "reason":"invalid_admission_offer","peer":peer}));
2902 }
2903 };
2904 guard
2905 .as_mut()
2906 .ok_or("node is closed")?
2907 .inbound_admission = Some(incoming);
2908 return Ok(staged);
2909 }
2910 let compact_pending = guard
2911 .as_ref()
2912 .ok_or("node is closed")?
2913 .pending_join
2914 .as_ref()
2915 .is_some_and(|pending| pending.admission_request().is_err());
2916 if compact_pending {
2917 if let Some(mut exchange) = guard
2918 .as_mut()
2919 .ok_or("node is closed")?
2920 .checkpoint_exchange
2921 .take()
2922 {
2923 if !exchange.task.is_finished() {
2924 let peer = exchange.peer;
2925 guard.as_mut().ok_or("node is closed")?.checkpoint_exchange = Some(exchange);
2926 return Ok(
2927 json!({"state":"admission_pending", "phase":"checkpoint", "peer":peer}),
2928 );
2929 }
2930 match guard
2931 .as_ref()
2932 .ok_or("node is closed")?
2933 .runtime
2934 .block_on(&mut exchange.task)
2935 .map_err(|_| "checkpoint exchange task cancelled")?
2936 {
2937 Ok(value) => {
2938 let checkpoint: Vec<u8> =
2939 serde_json::from_value(value["checkpoint"].clone())
2940 .map_err(|_| "checkpoint reply is invalid")?;
2941 let peer: [u8; 32] = serde_json::from_value(value["peer"].clone())
2942 .map_err(|_| "checkpoint peer is invalid")?;
2943 let session = guard.as_mut().ok_or("node is closed")?;
2944 session
2945 .pending_join
2946 .as_mut()
2947 .ok_or("session has no pending join")?
2948 .complete_checkpoint(&checkpoint)
2949 .map_err(str::to_owned)?;
2950 let lifecycle = session
2951 .join_lifecycle
2952 .as_mut()
2953 .ok_or("join lifecycle has no persisted Iroh peers")?;
2954 lifecycle.selected = Some(peer);
2955 lifecycle.cursor = lifecycle
2956 .peers
2957 .iter()
2958 .position(|candidate| *candidate == peer)
2959 .unwrap_or(lifecycle.cursor);
2960 persistence::commit_pending_join(session)?;
2961 }
2962 Err(reason) => {
2963 let session = guard.as_mut().ok_or("node is closed")?;
2964 let lifecycle = session
2965 .join_lifecycle
2966 .as_mut()
2967 .ok_or("join lifecycle has no persisted Iroh peers")?;
2968 if lifecycle.selected == Some(exchange.peer) {
2969 lifecycle.advance();
2970 persistence::commit_pending_join(session)?;
2971 }
2972 tracing::debug!(
2973 target: "data_fabric_transport",
2974 peer = ?exchange.peer,
2975 %reason,
2976 "CHECKPOINT_EXCHANGE_FAILED"
2977 );
2978 return Ok(json!({"state":admission_state::UNAVAILABLE,
2979 "reason":"checkpoint_unavailable", "peer":exchange.peer}));
2980 }
2981 }
2982 } else {
2983 let (peers, invitation, selected) = {
2984 let session = guard.as_mut().ok_or("node is closed")?;
2985 let lifecycle = session
2986 .join_lifecycle
2987 .as_ref()
2988 .ok_or("join lifecycle has no persisted Iroh peers")?;
2989 let mut peers = Vec::with_capacity(lifecycle.peers.len());
2990 if let Some(selected) = lifecycle.selected {
2991 peers.push(selected);
2992 }
2993 for peer in lifecycle.peers.iter().skip(lifecycle.cursor) {
2994 if !peers.contains(peer) {
2995 peers.push(*peer);
2996 }
2997 }
2998 let selected = *peers.first().ok_or("no reachable workspace member")?;
2999 let invitation = session
3000 .pending_join
3001 .as_ref()
3002 .ok_or("session has no pending join")?
3003 .deferred_invitation()
3004 .map_err(str::to_owned)?
3005 .to_vec();
3006 (peers, invitation, selected)
3007 };
3008 {
3009 let session = guard.as_mut().ok_or("node is closed")?;
3010 let lifecycle = session
3011 .join_lifecycle
3012 .as_mut()
3013 .ok_or("join lifecycle has no persisted Iroh peers")?;
3014 lifecycle.selected = Some(selected);
3015 persistence::commit_pending_join(session)?;
3016 }
3017 let wake = guard
3018 .as_ref()
3019 .ok_or("node is closed")?
3020 .node
3021 .control_signal();
3022 let client = guard
3023 .as_ref()
3024 .ok_or("node is closed")?
3025 .node
3026 .control_client();
3027 let requester = guard.as_ref().ok_or("node is closed")?.node.id();
3028 let task = guard
3029 .as_ref()
3030 .ok_or("node is closed")?
3031 .runtime
3032 .spawn(async move {
3033 let outcome =
3034 request_invitation_checkpoint(client, requester, invitation, peers)
3035 .await;
3036 wake.notify_one();
3037 outcome
3038 });
3039 guard.as_mut().ok_or("node is closed")?.checkpoint_exchange =
3040 Some(PendingCheckpointExchange {
3041 peer: selected,
3042 task,
3043 });
3044 return Ok(
3045 json!({"state":"admission_pending", "phase":"checkpoint", "peer":selected}),
3046 );
3047 }
3048 }
3049 let mut reply = None;
3050 if let Some(mut exchange) = guard.as_mut().ok_or("node is closed")?.join_exchange.take() {
3051 if !exchange.task.is_finished() {
3052 let peer = exchange.peer;
3053 guard.as_mut().ok_or("node is closed")?.join_exchange = Some(exchange);
3054 return Ok(json!({"state":"admission_pending", "peer":peer}));
3055 }
3056 let outcome = guard
3057 .as_ref()
3058 .ok_or("node is closed")?
3059 .runtime
3060 .block_on(&mut exchange.task)
3061 .map_err(|_| "join exchange task cancelled")?;
3062 match outcome {
3063 JoinAttemptOutcome::NotSent => {
3064 let session = guard.as_mut().ok_or("node is closed")?;
3065 let lifecycle = session
3066 .join_lifecycle
3067 .as_mut()
3068 .ok_or("join lifecycle has no persisted Iroh peers")?;
3069 if lifecycle.selected == Some(exchange.peer) {
3070 lifecycle.advance();
3071 persistence::commit_pending_join(session)?;
3072 }
3073 }
3074 JoinAttemptOutcome::Waiting => {
3075 return Ok(json!({"state":admission_state::WAITING, "peer":exchange.peer}));
3076 }
3077 JoinAttemptOutcome::Failed(reason) => {
3078 tracing::debug!(
3079 target: "data_fabric_transport",
3080 peer = ?exchange.peer,
3081 %reason,
3082 "JOIN_EXCHANGE_FAILED"
3083 );
3084 return Ok(json!({"state":admission_state::UNAVAILABLE,
3085 "reason":"invalid_admission_reply", "peer":exchange.peer}));
3086 }
3087 JoinAttemptOutcome::Reply {
3088 value,
3089 history_prefix,
3090 } => {
3091 let session = guard.as_mut().ok_or("node is closed")?;
3092 session.join_history_prefix = history_prefix;
3093 reply = Some(value);
3094 }
3095 }
3096 }
3097 if let Some(reply) = reply {
3098 if reply.get("state").is_some() {
3099 return Ok(reply);
3100 }
3101 let commits = if let Some(commits) = reply.get("commits") {
3102 serde_json::from_value(commits.clone())
3103 .map_err(|_| "admission reply commits are invalid")?
3104 } else {
3105 serde_json::from_value(json!([{
3106 "commit": reply["commit"].clone(),
3107 "authorization": reply["authorization"].clone(),
3108 }]))
3109 .map_err(|_| "admission reply is incomplete")?
3110 };
3111 let welcome = serde_json::from_value(reply["welcome"].clone())
3112 .map_err(|_| "admission reply has no Welcome")?;
3113 let staged = execute_in_session(guard, Request::StageJoin { commits, welcome }, ended)?;
3114 let snapshot: Vec<u8> = serde_json::from_value(staged["snapshot"].clone())
3115 .map_err(|_| "join candidate snapshot is invalid")?;
3116 {
3117 let session = guard.as_mut().ok_or("node is closed")?;
3118 persistence::commit_candidate(session, &snapshot)?;
3119 }
3120 let mut joined = execute_in_session(guard, Request::AdoptJoin { snapshot }, ended)?;
3121 joined["state"] = json!("workspace_joined");
3122 return Ok(joined);
3123 }
3124
3125 let (peer, request, name, checkpoint) = {
3126 let session = guard.as_mut().ok_or("node is closed")?;
3127 let lifecycle = session
3128 .join_lifecycle
3129 .as_mut()
3130 .ok_or("join lifecycle has no persisted Iroh peers")?;
3131 let peer = lifecycle
3132 .selected
3133 .or_else(|| lifecycle.peers.get(lifecycle.cursor).copied());
3134 let Some(peer) = peer else {
3135 return Ok(json!({"state":admission_state::UNAVAILABLE,
3136 "reason":"no_reachable_member"}));
3137 };
3138 if lifecycle.selected.is_none() {
3139 lifecycle.selected = Some(peer);
3140 persistence::commit_pending_join(session)?;
3141 }
3142 let pending = session
3143 .pending_join
3144 .as_ref()
3145 .ok_or("session has no pending join")?;
3146 (
3147 peer,
3148 pending.admission_request().map_err(str::to_owned)?.to_vec(),
3149 pending.member().display_name().as_bytes().to_vec(),
3150 pending
3151 .admission_checkpoint()
3152 .map_err(str::to_owned)?
3153 .to_vec(),
3154 )
3155 };
3156 let wake = guard
3157 .as_ref()
3158 .ok_or("node is closed")?
3159 .node
3160 .control_signal();
3161 let client = guard
3162 .as_ref()
3163 .ok_or("node is closed")?
3164 .node
3165 .control_client();
3166 let task = guard
3167 .as_ref()
3168 .ok_or("node is closed")?
3169 .runtime
3170 .spawn(async move {
3171 let outcome = request_join_exchange(client, peer, request, name, checkpoint).await;
3172 wake.notify_one();
3173 outcome
3174 });
3175 guard.as_mut().ok_or("node is closed")?.join_exchange =
3176 Some(PendingJoinExchange { peer, task });
3177 return Ok(json!({"state":"admission_pending", "peer":peer}));
3178 }
3179 let session = guard.as_mut().ok_or("node is closed")?;
3180 if matches!(request, Request::WorkspaceState {}) {
3181 let mut value = json!({
3182 "activity": activity_value(session),
3183 "workspace_ready": session.workspace.is_some(),
3184 "durable": session.records.is_some(),
3185 });
3186 if let Some(owner) = &session.workspace {
3187 value["workspace"] = json!(owner.id());
3188 } else if let Some(pending) = &session.pending_join {
3189 value["workspace"] = json!(pending.workspace_id());
3190 }
3191 return Ok(value);
3192 }
3193 if matches!(request, Request::WorkspaceMetrics {}) {
3194 let owner = session
3195 .workspace
3196 .as_ref()
3197 .ok_or("session has no workspace")?;
3198 let roster = owner.member_roster().map_err(str::to_owned)?;
3199 let metrics = session.node.transport_metrics();
3200 let paths: Vec<_> = metrics
3201 .paths
3202 .iter()
3203 .filter_map(|path| {
3204 let member = roster
3205 .iter()
3206 .find(|member| member.endpoint == path.endpoint)?;
3207 Some(json!({"member":member.id, "route":path.route, "rtt_ms":path.rtt_ms}))
3208 })
3209 .collect();
3210 return Ok(json!({
3211 "workspace":owner.id(), "session":session.node.id(),
3212 "received_bytes":metrics.received_bytes, "sent_bytes":metrics.sent_bytes,
3213 "receive_queue":metrics.receive_queue,
3214 "admission_queue":session.admission_queue.len(),
3215 "admission_queue_bytes":session.admission_queue.bytes(),
3216 "admission_waiters":session.admission_waiters.len(),
3217 "control_timing":session.node.control_timing(),
3218 "membership_gossip":session.gossip_counts.json(),
3219 "connection_capacity":session.node.connection_capacity(),
3220 "gossip_neighbors":session.runtime.block_on(session.node.live_neighbors(owner.id())).len(),
3221 "admission_in_flight":session.queued_admission_in_flight.len(),
3222 "approval_pending":session.pending_approvals.len(),
3223 "activity":activity_value(session),
3224 "pending_objects":session.inbox.as_ref().map(|inbox| inbox.pending_count()).unwrap_or(0) + session.recovered.len(),
3225 "repair_jobs":usize::from(session.cutoff.is_some()) + usize::from(session.range.is_some() || session.ready_range.is_some())
3226 + usize::from(session.direct_range.is_some() || session.ready_direct_range.is_some())
3227 + usize::from(session.current_view.is_some() || session.ready_current_view.is_some()),
3228 "paths":paths, "paths_limited":metrics.paths_limited
3229 }));
3230 }
3231 if matches!(request, Request::NetworkChange {}) {
3232 session.runtime.block_on(session.node.network_change());
3233 session.interests.repair();
3234 return Ok(json!({"notified":true}));
3235 }
3236 if matches!(request, Request::NearbyEndpoints {}) {
3237 let peers = session.runtime.block_on(session.node.nearby_peers());
3238 let names = session.runtime.block_on(async {
3239 let mut queries = tokio::task::JoinSet::new();
3240 for peer in &peers {
3241 let peer = *peer;
3242 let request = session.node.request_control(peer, NEARBY_IDENTITY);
3243 queries.spawn(async move {
3244 tokio::time::timeout(Duration::from_millis(750), request)
3245 .await
3246 .ok()
3247 .and_then(Result::ok)
3248 .and_then(|reply| String::from_utf8(reply).ok())
3249 .map(|name| json!({"id": peer, "name": name}))
3250 });
3251 }
3252 let mut names = Vec::new();
3253 while let Some(Ok(Some(name))) = queries.join_next().await {
3254 names.push(name);
3255 }
3256 names
3257 });
3258 return Ok(json!({"endpoints":peers,"names":names}));
3259 }
3260 if matches!(request, Request::NearbyWorkspaces {}) {
3261 let peers = session
3262 .runtime
3263 .block_on(session.node.nearby_workspace_peers());
3264 let peer_count = peers.len();
3265 let (found, checked) = session.runtime.block_on(async {
3266 let mut queries = tokio::task::JoinSet::new();
3267 for peer in peers {
3268 let request = session.node.request_control(peer, NEARBY_WORKSPACE);
3269 queries.spawn(async move {
3270 tokio::time::timeout(Duration::from_secs(3), request)
3271 .await
3272 .ok()
3273 .and_then(Result::ok)
3274 .map(|reply| (peer, reply))
3275 });
3276 }
3277 let mut advertisements = BTreeMap::<Vec<u8>, Value>::new();
3278 let mut checked = 0usize;
3279 while let Some(Ok(Some((peer, reply)))) = queries.join_next().await {
3280 checked += 1;
3281 for payload in nearby_workspace_payloads(&reply) {
3282 let Some((&mode, payload)) = payload.split_first() else {
3283 continue;
3284 };
3285 let (workspace_name, invitation) = match mode {
3286 1 | 2 => (None, payload),
3287 3 | 4 if payload.len() >= 2 => {
3288 let length = u16::from_be_bytes([payload[0], payload[1]]) as usize;
3289 if !(1..=320).contains(&length) || payload.len() < length + 2 { continue; }
3290 let Ok(name) = std::str::from_utf8(&payload[2..2 + length]) else { continue; };
3291 if arachne_security::validate_workspace_name(name).is_err() { continue; }
3292 (Some(name), &payload[2 + length..])
3293 }
3294 _ => continue,
3295 };
3296 if invitation.is_empty() || invitation.len() > 2048 {
3297 continue;
3298 }
3299 advertisements
3300 .entry(invitation.to_vec())
3301 .or_insert_with(|| {
3302 json!({
3303 "peer": peer,
3304 "mode": if mode == 1 || mode == 3 { "request_access" } else { "open_joining" },
3305 "workspace_name": workspace_name,
3306 "invitation": invitation,
3307 })
3308 });
3309 }
3310 }
3311 (advertisements.into_values().collect::<Vec<_>>(), checked)
3312 });
3313 return Ok(json!({"workspaces":found,"endpoints_checked":checked,
3314 "limited":peer_count == 16 || checked < peer_count}));
3315 }
3316 if let Request::SetNearbyWorkspace {
3317 mode,
3318 invitation,
3319 workspace_name,
3320 workspace,
3321 } = &request
3322 {
3323 if let Some(name) = workspace_name {
3324 arachne_security::validate_workspace_name(name).map_err(str::to_owned)?;
3325 }
3326 let encoded = match mode.as_deref() {
3327 None if invitation.is_empty() && workspace_name.is_none() => None,
3328 Some("request_access" | "open_joining")
3329 if !invitation.is_empty() && invitation.len() <= 2048 =>
3330 {
3331 let mut value = vec![
3332 if mode.as_deref() == Some("request_access") {
3333 1
3334 } else {
3335 2
3336 } + if workspace_name.is_some() { 2 } else { 0 },
3337 ];
3338 if let Some(name) = workspace_name {
3339 value.extend((name.len() as u16).to_be_bytes());
3340 value.extend(name.as_bytes());
3341 }
3342 value.extend(invitation);
3343 Some(value)
3344 }
3345 _ => return Err("invalid nearby workspace advertisement".into()),
3346 };
3347 if let Some(encoded) = encoded {
3348 let key = nearby_workspace_key(*workspace, invitation);
3349 if !session.nearby_workspaces.contains_key(&key)
3350 && session.nearby_workspaces.len() >= MAX_NEARBY_WORKSPACES
3351 {
3352 return Err("nearby workspace advertisement limit reached".into());
3353 }
3354 session.nearby_workspaces.insert(key, encoded);
3355 } else if let Some(workspace) = workspace {
3356 session.nearby_workspaces.remove(workspace);
3357 } else {
3358 session.nearby_workspaces.clear();
3359 }
3360 return Ok(json!({"state":if session.nearby_workspaces.is_empty() {
3361 "nearby_workspace_private"
3362 } else {
3363 "nearby_workspace_advertised"
3364 }}));
3365 }
3366 if let Request::SetNearbyIdentity { name } = &request {
3367 arachne_security::validate_workspace_name(name).map_err(str::to_owned)?;
3368 session.nearby_identity = Some(name.trim().to_owned());
3369 return Ok(json!({"state":"nearby_identity_set"}));
3370 }
3371 if let Request::SendNearbyInvitation { peer, invitation } = &request {
3372 if invitation.is_empty() || invitation.len() > 2048 {
3373 return Err("invalid nearby invitation length".into());
3374 }
3375 let mut packet = NEARBY_INVITATION.to_vec();
3376 packet.extend((invitation.len() as u16).to_be_bytes());
3377 packet.extend(invitation);
3378 let reply = session
3379 .runtime
3380 .block_on(session.node.request_control(*peer, &packet))
3381 .map_err(|error| error.to_string())?;
3382 if reply != [1] {
3383 return Err("nearby device rejected invitation".into());
3384 }
3385 return Ok(json!({"state":"nearby_invitation_sent","peer":peer}));
3386 }
3387 if let Some((removed, expected)) = &session.staged_removal {
3388 let Request::AdoptAdmission { snapshot } = request else {
3389 return Err("removed membership awaits durable adoption".into());
3390 };
3391 if &snapshot != expected {
3392 return Err("removed snapshot does not match candidate".into());
3393 }
3394 if let Some(store) = &session.records {
3395 store.require_committed(&snapshot)?;
3396 }
3397 let value = json!({"workspace":removed.workspace_id(), "epoch":removed.epoch(),
3398 "state":"removed", "workspace_ready":false, "member":{"id":removed.member().id(),
3399 "display_name":removed.member().display_name()}, "commit_digest":removed.commit_digest()});
3400 *ended = Some(guard.take().ok_or("node is closed")?);
3401 return Ok(value);
3402 }
3403 if !session.recovered.is_empty() && !matches!(request, Request::PollRecoveredPublication {}) {
3404 return Err("drain adopted recovery publications before another operation".into());
3405 }
3406 if session.staged_workspace.is_some()
3407 && !matches!(
3408 request,
3409 Request::AdoptAdmission { .. }
3410 | Request::AdoptJoin { .. }
3411 | Request::AdoptPublication { .. }
3412 | Request::AdoptReception { .. }
3413 | Request::AdoptRecovery { .. }
3414 | Request::AdoptCurrentView { .. }
3415 | Request::PollAdmission { .. }
3416 | Request::OfferStagedMembershipUpdate { .. }
3417 | Request::PollMembershipOffer {}
3418 | Request::DiscardWorkspaceCandidate {}
3419 )
3420 {
3421 return Err(
3422 "workspace candidate awaits durable adoption; close and restore saved state to recover"
3423 .into(),
3424 );
3425 }
3426 if session.inbound_admission.is_some()
3427 && !matches!(
3428 request,
3429 Request::AdoptAdmission { .. }
3430 | Request::AdoptJoin { .. }
3431 | Request::SendAdmissionReply {}
3432 | Request::PollAdmission { .. }
3433 )
3434 {
3435 return Err("received admission awaits adoption or reply; close to recover".into());
3436 }
3437 if session.inbox.is_none()
3440 && (session.cutoff.is_some() || session.range.is_some() || session.ready_range.is_some())
3441 {
3442 match request {
3443 Request::PollProtected {} => return Ok(Value::Null),
3444 Request::StageReception { .. } => {
3445 return Err("recovery must finish before live reception".into());
3446 }
3447 _ => (),
3448 }
3449 }
3450 if let Request::Resource { request } = request {
3451 return resources::execute(session, request);
3452 }
3453 if session.workspace.is_some() && matches!(request, Request::Publish { .. }) {
3454 return Err("unprotected publication is disabled for an admitted workspace".into());
3455 }
3456 if session.workspace.is_some() && matches!(request, Request::InstallVerifiedPolicy { .. }) {
3457 return Err("admitted workspace routing must derive from verified membership".into());
3458 }
3459 if session.workspace.is_some() && matches!(request, Request::Poll {}) {
3460 return Err("use poll_protected for an admitted workspace".into());
3461 }
3462 let value = if matches!(request, Request::EndpointInfo {}) {
3463 json!({"endpoint_key":session.node.id(), "bound_address":session.node.address().to_string()})
3464 } else if let Request::ControlExchange {
3465 peer,
3466 address,
3467 payload,
3468 } = request
3469 {
3470 if let Some(address) = address {
3471 let address: std::net::SocketAddr =
3472 address.parse().map_err(|_| "invalid address hint")?;
3473 session
3474 .runtime
3475 .block_on(session.node.add_address_hint(peer, address))
3476 .map_err(|e| e.to_string())?;
3477 }
3478 let reply = session
3479 .runtime
3480 .block_on(session.node.request_control(peer, &payload))
3481 .map_err(|e| e.to_string())?;
3482 json!({"reply": reply})
3483 } else if let Request::PollWorkspacePresence { announce } = request {
3484 presence::poll(session, announce)?
3485 } else if matches!(
3486 request,
3487 Request::FetchMembershipUpdate { .. }
3488 | Request::PollMembershipUpdate {}
3489 | Request::NextMembershipPeer { .. }
3490 | Request::OfferMembershipUpdate { .. }
3491 | Request::OfferStagedMembershipUpdate { .. }
3492 | Request::PollMembershipOffer {}
3493 ) {
3494 membership::poll(session, request)?
3495 } else if matches!(
3496 request,
3497 Request::EnableObjectDelivery {}
3498 | Request::PollPendingObject { .. }
3499 | Request::StageObjectAcknowledgement { .. }
3500 | Request::StageObjectRejection { .. }
3501 ) {
3502 protected::inbox_operation(session, request)?
3503 } else if matches!(
3504 request,
3505 Request::StageNetworkPublication { .. } | Request::PollProtected {}
3506 ) {
3507 protected::stage(session, request)?
3508 } else if matches!(
3509 request,
3510 Request::StagePublication { .. } | Request::StageReception { .. }
3511 ) {
3512 if session.inbox.is_some() {
3513 return Err("raw MLS applications disabled after object cutover".into());
3514 }
3515 let original = session
3516 .workspace
3517 .as_ref()
3518 .ok_or("session has no workspace")?;
3519 let key = session
3520 .storage_key
3521 .as_ref()
3522 .ok_or("session has no protected root key")?;
3523 let mut candidate = original.provisional_copy().map_err(str::to_owned)?;
3524 let (transition, state) = match request {
3525 Request::StagePublication { context, payload } => (
3526 WorkspaceTransition::Publication(
3527 candidate
3528 .protect_application(&context, &payload)
3529 .map_err(str::to_owned)?,
3530 ),
3531 "awaiting_publication_save",
3532 ),
3533 Request::StageReception {
3534 context,
3535 ciphertext,
3536 } => (
3537 WorkspaceTransition::Reception(
3538 candidate
3539 .unprotect_application(&context, &ciphertext)
3540 .map_err(str::to_owned)?,
3541 ),
3542 "awaiting_reception_save",
3543 ),
3544 _ => unreachable!(),
3545 };
3546 let snapshot = seal_state(
3547 session.records.is_some(),
3548 &candidate,
3549 key,
3550 session.publisher.as_ref(),
3551 session.received.as_ref(),
3552 session.inbox.as_ref(),
3553 )?;
3554 let value = json!({"workspace":candidate.id(), "snapshot":snapshot, "state":state, "durable":false});
3555 session.staged_workspace = Some(StagedWorkspace {
3556 publisher: session.publisher.clone(),
3557 received: session.received.clone(),
3558 inbox: session.inbox.clone(),
3559 transition,
3560 workspace: candidate,
3561 snapshot,
3562 });
3563 value
3564 } else if let Request::StageRecoveryRange { retain_until } = request {
3565 protected::stage_recovery(session, retain_until)?
3566 } else if matches!(request, Request::PollRecoveredPublication {}) {
3567 match session.recovered.pop_front() {
3568 Some((context, message)) => json!({"workspace":context.workspace,
3569 "revision":context.revision, "topic":context.topic.as_str(), "id":context.id,
3570 "sequence":context.sequence.map(|n| n.get()),
3571 "payload":message.payload, "member":message.member, "endpoint":message.endpoint}),
3572 None => Value::Null,
3573 }
3574 } else if let Request::FetchRecoveryRange {
3575 peer,
3576 author,
3577 revision,
3578 topics,
3579 after,
3580 through,
3581 } = request
3582 {
3583 if session.cutoff.is_some()
3584 || session.range.is_some()
3585 || session.ready_range.is_some()
3586 || session.direct_range.is_some()
3587 || session.ready_direct_range.is_some()
3588 || session.current_view.is_some()
3589 || session.ready_current_view.is_some()
3590 {
3591 return Err("recovery operation already pending".into());
3592 }
3593 let owner = session
3594 .workspace
3595 .as_ref()
3596 .ok_or("session has no workspace")?;
3597 if peer == Some(session.node.id())
3598 || topics.is_empty()
3599 || topics.len() > arachne_delivery::MAX_TOPICS
3600 || matches!((after, through), (Some(after), Some(through)) if after >= through)
3601 || matches!((after, through), (Some(_), None) | (None, Some(_)))
3602 {
3603 return Err("invalid recovery peer, selection or range".into());
3604 }
3605 let count = topics.len();
3606 let topics = topics
3607 .into_iter()
3608 .map(Topic::new)
3609 .collect::<Result<BTreeSet<_>, _>>()
3610 .map_err(|e| e.to_string())?;
3611 if topics.len() != count {
3612 return Err("duplicate recovery topic".into());
3613 }
3614 let author = match (author, peer) {
3615 (Some(author), _) => owner
3616 .endpoints_for_members(&[author])
3617 .map(|_| author)
3618 .map_err(str::to_owned),
3619 (None, Some(peer)) => owner.member_id_for_endpoint(peer).map_err(str::to_owned),
3620 (None, None) => Err("automatic recovery requires an original author".into()),
3621 }?;
3622 let after = match after {
3623 Some(after) => after,
3624 None => session
3625 .inbox
3626 .as_ref()
3627 .ok_or("automatic recovery requires object delivery")?
3628 .recovery_progress(author, &topics),
3629 };
3630 let available = through
3631 .is_none()
3632 .then(|| arachne_delivery::wire::AvailableRangeQuery {
3633 workspace: owner.id(),
3634 author,
3635 epoch: owner.epoch(),
3636 policy_revision: revision,
3637 after,
3638 topics: topics.clone(),
3639 });
3640 let query = arachne_delivery::RangeQuery {
3641 workspace: owner.id(),
3642 author,
3643 epoch: owner.epoch(),
3644 policy_revision: revision,
3645 after,
3646 through: through.unwrap_or_else(|| after.saturating_add(1)),
3647 topics,
3648 };
3649 if query.after == query.through {
3650 return Err("recovery cursor exhausted".into());
3651 }
3652 let automatic = available.is_some() || peer.is_none();
3653 let mut candidates = if automatic {
3654 session
3655 .runtime
3656 .block_on(session.node.live_neighbors(query.workspace))
3657 } else {
3658 vec![peer.unwrap()]
3659 };
3660 if automatic {
3661 candidates.sort_unstable();
3662 candidates.dedup();
3663 candidates.truncate(MAX_WORKSPACE_OVERLAY_PATHS);
3664 let author_endpoint = owner
3665 .endpoints_for_members(&[query.author])
3666 .map_err(str::to_owned)?[0];
3667 if author_endpoint != session.node.id()
3668 && !candidates.contains(&author_endpoint)
3669 && session
3670 .runtime
3671 .block_on(session.node.address_hint(author_endpoint))
3672 .is_some()
3673 {
3674 candidates.insert(0, author_endpoint);
3675 }
3676 candidates.retain(|candidate| {
3677 check_recovery_policy(
3678 session,
3679 *candidate,
3680 query.author,
3681 query.workspace,
3682 query.epoch,
3683 revision,
3684 &query.topics,
3685 )
3686 .is_ok()
3687 });
3688 } else {
3689 check_recovery_policy(
3690 session,
3691 candidates[0],
3692 query.author,
3693 query.workspace,
3694 query.epoch,
3695 revision,
3696 &query.topics,
3697 )?;
3698 }
3699 if candidates.is_empty() {
3700 return Ok(json!({"state":"recovery_source_waiting", "accepted_progress":false}));
3701 }
3702 let wire = match &available {
3703 Some(request) => request.to_wire(),
3704 None => query.to_wire(),
3705 }
3706 .map_err(str::to_owned)?;
3707 let requests = candidates
3708 .iter()
3709 .map(|peer| (*peer, session.node.request_control(*peer, &wire)))
3710 .collect::<Vec<_>>();
3711 let candidate_count = requests.len();
3712 let (reply_tx, replies) = mpsc::channel(candidate_count);
3713 let task = session.runtime.spawn(async move {
3714 let mut pending = tokio::task::JoinSet::new();
3715 for (peer, request) in requests {
3716 pending.spawn(async move { (peer, request.await.map_err(|e| e.to_string())) });
3717 }
3718 while let Some(result) = pending.join_next().await {
3719 if let Ok(reply) = result
3720 && reply_tx.send(reply).await.is_err()
3721 {
3722 break;
3723 }
3724 }
3725 });
3726 session.range = Some(PendingRange {
3727 query,
3728 available,
3729 automatic,
3730 replies,
3731 task,
3732 attempted: 0,
3733 reason: None,
3734 });
3735 json!({"state":"recovery_range_pending", "candidate_count":candidate_count,
3736 "automatic_source":automatic, "accepted_progress":false})
3737 } else if matches!(request, Request::CancelRecoveryRange {}) {
3738 drop(session.range.take());
3739 session.ready_range = None;
3740 json!({"state":"recovery_range_cancelled", "accepted_progress":false})
3741 } else if matches!(request, Request::PollRecoveryRange {}) {
3742 let Some(active) = session.range.as_mut() else {
3743 return Ok(Value::Null);
3744 };
3745 let (peer, reply) = match active.replies.try_recv() {
3746 Ok(reply) => reply,
3747 Err(mpsc::error::TryRecvError::Empty) if !active.task.is_finished() => {
3748 return Ok(Value::Null);
3749 }
3750 Err(_) => {
3751 let mut pending = session.range.take().unwrap();
3752 if pending.automatic {
3753 return Ok(json!({"state":"recovery_source_unavailable",
3754 "attempted":pending.attempted,
3755 "reason":pending.reason.take().unwrap_or_else(|| "no current holder supplied authenticated coverage".into()),
3756 "automatic_source":true, "accepted_progress":false}));
3757 }
3758 return Err(pending
3759 .reason
3760 .take()
3761 .unwrap_or_else(|| "recovery request failed".into()));
3762 }
3763 };
3764 active.attempted += 1;
3765 let query = active.query.clone();
3766 let available = active.available.clone();
3767 let automatic = active.automatic;
3768 let attempted = active.attempted;
3769 let owner = session
3770 .workspace
3771 .as_ref()
3772 .ok_or("session has no workspace")?;
3773 if !automatic {
3774 drop(session.range.take());
3775 check_recovery_policy(
3776 session,
3777 peer,
3778 query.author,
3779 query.workspace,
3780 query.epoch,
3781 query.policy_revision,
3782 &query.topics,
3783 )?;
3784 let reply = reply?;
3785 match arachne_delivery::wire::verify_reply(owner, &query, &reply)
3786 .map_err(str::to_owned)?
3787 {
3788 arachne_delivery::wire::RangeReply::Rejected(error) => {
3789 json!({"state":"recovery_range_rejected", "reason":error.to_string(), "accepted_progress":false})
3790 }
3791 arachne_delivery::wire::RangeReply::Offered(range) => {
3792 let packet_count = range.packets().len();
3793 drop(range);
3794 session.ready_range = Some(ReadyRange {
3795 query,
3796 peer,
3797 reply,
3798 packet_count,
3799 automatic: false,
3800 });
3801 let ready = session.ready_range.as_ref().unwrap();
3802 json!({"state":"recovery_range_ready", "workspace":ready.query.workspace, "author":ready.query.author,
3803 "peer":ready.peer, "epoch":ready.query.epoch, "revision":ready.query.policy_revision,
3804 "after":ready.query.after, "through":ready.query.through, "packet_count":ready.packet_count,
3805 "retained_bytes":ready.reply.len(), "automatic_source":false, "accepted_progress":false})
3806 }
3807 }
3808 } else {
3809 let result = check_recovery_policy(
3810 session,
3811 peer,
3812 query.author,
3813 query.workspace,
3814 query.epoch,
3815 query.policy_revision,
3816 &query.topics,
3817 )
3818 .and_then(|_| {
3819 reply
3820 .map_err(|_| "reachable holder did not answer".into())
3821 .and_then(|reply| {
3822 let (query, reply) = match &available {
3823 Some(request) => {
3824 arachne_delivery::wire::parse_available_reply(request, &reply)
3825 .map_err(|_| {
3826 "holder returned invalid recovery evidence".to_string()
3827 })?
3828 .ok_or_else(|| "holder has no retained range".to_string())?
3829 }
3830 None => (query.clone(), reply),
3831 };
3832 match arachne_delivery::wire::verify_reply(owner, &query, &reply) {
3833 Ok(arachne_delivery::wire::RangeReply::Offered(range)) => {
3834 Ok((query, reply, range.packets().len()))
3835 }
3836 Ok(arachne_delivery::wire::RangeReply::Rejected(error)) => {
3837 Err(error.to_string())
3838 }
3839 Err(_) => Err("holder returned invalid recovery evidence".into()),
3840 }
3841 })
3842 });
3843 if let Ok((query, reply, packet_count)) = result {
3844 drop(session.range.take());
3845 session.ready_range = Some(ReadyRange {
3846 query,
3847 peer,
3848 reply,
3849 packet_count,
3850 automatic: true,
3851 });
3852 let ready = session.ready_range.as_ref().unwrap();
3853 json!({"state":"recovery_range_ready", "workspace":ready.query.workspace, "author":ready.query.author,
3854 "peer":ready.peer, "epoch":ready.query.epoch, "revision":ready.query.policy_revision,
3855 "after":ready.query.after, "through":ready.query.through, "packet_count":ready.packet_count,
3856 "retained_bytes":ready.reply.len(), "automatic_source":true, "attempted":attempted,
3857 "accepted_progress":false})
3858 } else {
3859 session.range.as_mut().unwrap().reason = result.err();
3860 Value::Null
3861 }
3862 }
3863 } else if matches!(request, Request::NextDirectGap {}) {
3864 match session
3865 .inbox
3866 .as_ref()
3867 .ok_or("object delivery not enabled")?
3868 .next_direct_gap(
3869 session
3870 .workspace
3871 .as_ref()
3872 .ok_or("session has no workspace")?,
3873 )
3874 .map_err(str::to_owned)?
3875 {
3876 Some(gap) => json!({"state":"direct_recovery_needed", "author":gap.author,
3877 "revision":gap.revision, "topic":gap.topic.as_str(),
3878 "recipients":gap.recipients, "after":gap.after, "through":gap.through}),
3879 None => Value::Null,
3880 }
3881 } else if let Request::FetchDirectRecovery {
3882 author,
3883 revision,
3884 topic,
3885 recipients,
3886 after,
3887 through,
3888 } = request
3889 {
3890 if session.cutoff.is_some()
3891 || session.range.is_some()
3892 || session.ready_range.is_some()
3893 || session.direct_range.is_some()
3894 || session.ready_direct_range.is_some()
3895 || session.current_view.is_some()
3896 || session.ready_current_view.is_some()
3897 {
3898 return Err("continuity operation already pending".into());
3899 }
3900 let owner = session
3901 .workspace
3902 .as_ref()
3903 .ok_or("session has no workspace")?;
3904 let local = owner.member().ok_or("member required")?.id();
3905 if session.inbox.is_none()
3906 || !recipients.contains(&local)
3907 || recipients.len() > 64
3908 || recipients.windows(2).any(|pair| pair[0] >= pair[1])
3909 {
3910 return Err("invalid direct recovery audience".into());
3911 }
3912 let query = arachne_delivery::wire::DirectRangeQuery {
3913 workspace: owner.id(),
3914 author,
3915 epoch: owner.epoch(),
3916 policy_revision: revision,
3917 topic: Topic::new(topic).map_err(|e| e.to_string())?,
3918 recipients,
3919 after,
3920 through,
3921 };
3922 session.direct_miss = None;
3923 let wire = query.to_wire().map_err(str::to_owned)?;
3924 let topics = BTreeSet::from([query.topic.clone()]);
3925 let mut candidates = owner
3926 .endpoints_for_members(&query.recipients)
3927 .map_err(str::to_owned)?;
3928 candidates.extend(
3929 owner
3930 .endpoints_for_members(&[query.author])
3931 .map_err(str::to_owned)?,
3932 );
3933 candidates.sort_unstable();
3934 candidates.dedup();
3935 candidates.retain(|candidate| {
3936 *candidate != session.node.id()
3937 && check_recovery_policy(
3938 session,
3939 *candidate,
3940 query.author,
3941 query.workspace,
3942 query.epoch,
3943 query.policy_revision,
3944 &topics,
3945 )
3946 .is_ok()
3947 });
3948 if candidates.is_empty() {
3949 return Ok(json!({"state":"direct_recovery_source_waiting",
3950 "accepted_progress":false}));
3951 }
3952 let requests = candidates
3953 .iter()
3954 .map(|peer| (*peer, session.node.request_control(*peer, &wire)))
3955 .collect::<Vec<_>>();
3956 let candidate_count = requests.len();
3957 let (reply_tx, replies) = mpsc::channel(candidate_count);
3958 let task = session.runtime.spawn(async move {
3959 let mut pending = tokio::task::JoinSet::new();
3960 for (peer, request) in requests {
3961 pending.spawn(async move { (peer, request.await.map_err(|e| e.to_string())) });
3962 }
3963 while let Some(result) = pending.join_next().await {
3964 if let Ok(reply) = result
3965 && reply_tx.send(reply).await.is_err()
3966 {
3967 break;
3968 }
3969 }
3970 });
3971 session.direct_range = Some(PendingDirectRange {
3972 query,
3973 replies,
3974 task,
3975 attempted: 0,
3976 reason: None,
3977 });
3978 json!({"state":"direct_recovery_pending", "candidate_count":candidate_count,
3979 "accepted_progress":false})
3980 } else if matches!(request, Request::CancelDirectRecovery {}) {
3981 drop(session.direct_range.take());
3982 session.ready_direct_range = None;
3983 session.direct_miss = None;
3984 json!({"state":"direct_recovery_cancelled", "accepted_progress":false})
3985 } else if matches!(request, Request::PollDirectRecovery {}) {
3986 let Some(active) = session.direct_range.as_mut() else {
3987 return Ok(Value::Null);
3988 };
3989 let (peer, reply) = match active.replies.try_recv() {
3990 Ok(reply) => reply,
3991 Err(mpsc::error::TryRecvError::Empty) if !active.task.is_finished() => {
3992 return Ok(Value::Null);
3993 }
3994 Err(_) => {
3995 let mut pending = session.direct_range.take().unwrap();
3996 session.direct_miss = Some(pending.query.clone());
3997 return Ok(json!({"state":"direct_recovery_source_unavailable",
3998 "attempted":pending.attempted,
3999 "reason":pending.reason.take().unwrap_or_else(||
4000 "no intended recipient supplied the missing range".into()),
4001 "accepted_progress":false}));
4002 }
4003 };
4004 active.attempted += 1;
4005 let query = active.query.clone();
4006 let attempted = active.attempted;
4007 let result = reply.and_then(|reply| {
4008 let owner = session
4009 .workspace
4010 .as_ref()
4011 .ok_or("session has no workspace")?;
4012 check_recovery_policy(
4013 session,
4014 peer,
4015 query.author,
4016 query.workspace,
4017 query.epoch,
4018 query.policy_revision,
4019 &BTreeSet::from([query.topic.clone()]),
4020 )?;
4021 match arachne_delivery::wire::verify_direct_reply(owner, &query, &reply)
4022 .map_err(str::to_owned)?
4023 {
4024 arachne_delivery::wire::DirectRangeReply::Offered(packets) => {
4025 Ok((reply, packets.len()))
4026 }
4027 arachne_delivery::wire::DirectRangeReply::Unavailable => {
4028 Err("intended recipient has no retained range".into())
4029 }
4030 }
4031 });
4032 if let Ok((reply, packet_count)) = result {
4033 drop(session.direct_range.take());
4034 session.ready_direct_range = Some(ReadyDirectRange {
4035 query,
4036 peer,
4037 reply,
4038 packet_count,
4039 });
4040 let ready = session.ready_direct_range.as_ref().unwrap();
4041 json!({"state":"direct_recovery_ready", "workspace":ready.query.workspace,
4042 "author":ready.query.author, "peer":ready.peer, "epoch":ready.query.epoch,
4043 "revision":ready.query.policy_revision, "topic":ready.query.topic.as_str(),
4044 "after":ready.query.after, "through":ready.query.through,
4045 "packet_count":ready.packet_count, "retained_bytes":ready.reply.len(),
4046 "attempted":attempted, "accepted_progress":false})
4047 } else {
4048 session.direct_range.as_mut().unwrap().reason = result.err();
4049 Value::Null
4050 }
4051 } else if matches!(request, Request::StageDirectRecovery {}) {
4052 protected::stage_direct_recovery(session)?
4053 } else if matches!(request, Request::StageDirectMiss {}) {
4054 protected::stage_direct_miss(session)?
4055 } else if let Request::FetchCurrentView {
4056 peer,
4057 authority,
4058 revision,
4059 topic,
4060 selector,
4061 } = request
4062 {
4063 if session.current_view.is_some()
4064 || session.ready_current_view.is_some()
4065 || session.cutoff.is_some()
4066 || session.range.is_some()
4067 || session.ready_range.is_some()
4068 || session.direct_range.is_some()
4069 || session.ready_direct_range.is_some()
4070 {
4071 return Err("continuity operation already pending".into());
4072 }
4073 if peer == Some(session.node.id()) || session.inbox.is_none() {
4074 return Err("invalid current-view request".into());
4075 }
4076 let owner = session
4077 .workspace
4078 .as_ref()
4079 .ok_or("session has no workspace")?;
4080 let topic = Topic::new(topic).map_err(|e| e.to_string())?;
4081 let topics = BTreeSet::from([topic.clone()]);
4082 let authority_endpoint = owner
4083 .endpoints_for_members(&[authority])
4084 .map_err(str::to_owned)?[0];
4085 let query = arachne_delivery::current::CurrentViewQuery {
4086 workspace: owner.id(),
4087 authority,
4088 epoch: owner.epoch(),
4089 policy_revision: revision,
4090 topic,
4091 selector,
4092 };
4093 let automatic = peer.is_none();
4094 let mut candidates = match peer {
4095 Some(peer) => vec![peer],
4096 None => session
4097 .runtime
4098 .block_on(session.node.live_neighbors(query.workspace)),
4099 };
4100 if automatic {
4101 candidates.sort_unstable();
4102 candidates.dedup();
4103 candidates.truncate(MAX_WORKSPACE_OVERLAY_PATHS);
4104 if authority_endpoint != session.node.id()
4105 && !candidates.contains(&authority_endpoint)
4106 && session
4107 .runtime
4108 .block_on(session.node.address_hint(authority_endpoint))
4109 .is_some()
4110 {
4111 candidates.insert(0, authority_endpoint);
4112 }
4113 candidates.retain(|candidate| {
4114 check_recovery_policy(
4115 session,
4116 *candidate,
4117 authority,
4118 query.workspace,
4119 query.epoch,
4120 revision,
4121 &topics,
4122 )
4123 .is_ok()
4124 });
4125 } else {
4126 check_recovery_policy(
4127 session,
4128 candidates[0],
4129 authority,
4130 query.workspace,
4131 query.epoch,
4132 revision,
4133 &topics,
4134 )?;
4135 }
4136 if candidates.is_empty() {
4137 return Ok(json!({"state":"current_view_source_waiting", "accepted_progress":false}));
4138 }
4139 let wire = query.to_wire().map_err(str::to_owned)?;
4140 let requests = candidates
4141 .iter()
4142 .map(|peer| (*peer, session.node.request_control(*peer, &wire)))
4143 .collect::<Vec<_>>();
4144 let candidate_count = requests.len();
4145 let (reply_tx, replies) = mpsc::channel(candidate_count);
4146 let task = session.runtime.spawn(async move {
4147 let mut pending = tokio::task::JoinSet::new();
4148 for (peer, request) in requests {
4149 pending.spawn(async move { (peer, request.await.map_err(|e| e.to_string())) });
4150 }
4151 while let Some(result) = pending.join_next().await {
4152 if let Ok(reply) = result
4153 && reply_tx.send(reply).await.is_err()
4154 {
4155 break;
4156 }
4157 }
4158 });
4159 session.current_view = Some(PendingCurrentView {
4160 query,
4161 automatic,
4162 replies,
4163 task,
4164 attempted: 0,
4165 reason: None,
4166 best: None,
4167 });
4168 json!({"state":"current_view_pending", "candidate_count":candidate_count,
4169 "automatic_source":automatic, "accepted_progress":false})
4170 } else if matches!(request, Request::PollCurrentView {}) {
4171 let Some(mut active) = session.current_view.take() else {
4172 return Ok(Value::Null);
4173 };
4174 let query = active.query.clone();
4175 let automatic = active.automatic;
4176 let topics = BTreeSet::from([query.topic.clone()]);
4177 loop {
4178 match active.replies.try_recv() {
4179 Ok((peer, reply)) => {
4180 active.attempted += 1;
4181 let result = check_recovery_policy(
4182 session,
4183 peer,
4184 query.authority,
4185 query.workspace,
4186 query.epoch,
4187 query.policy_revision,
4188 &topics,
4189 )
4190 .and_then(|_| reply.map_err(|_| "reachable holder did not answer".into()))
4191 .and_then(|reply| {
4192 let owner = session
4193 .workspace
4194 .as_ref()
4195 .ok_or("session has no workspace")?;
4196 arachne_delivery::current::verify_wire_reply(owner, &query, &reply)
4197 .map_err(|_| "holder returned invalid current-view evidence".into())
4198 .and_then(|view| {
4199 view.map(|view| (reply, view))
4200 .ok_or("holder has no current view".into())
4201 })
4202 });
4203 match result {
4204 Ok((reply, view)) => {
4205 let candidate = ReadyCurrentView {
4206 query: query.clone(),
4207 peer,
4208 cut: view.cut,
4209 value_count: view.values.len(),
4210 reply,
4211 };
4212 if active.best.as_ref().is_none_or(|best| {
4213 (candidate.cut, &candidate.reply) > (best.cut, &best.reply)
4214 }) {
4215 active.best = Some(candidate);
4216 }
4217 }
4218 Err(reason) => active.reason = Some(reason),
4219 }
4220 }
4221 Err(mpsc::error::TryRecvError::Empty) if !active.task.is_finished() => {
4222 session.current_view = Some(active);
4223 return Ok(Value::Null);
4224 }
4225 Err(_) => break,
4226 }
4227 }
4228 let attempted = active.attempted;
4229 if let Some(ready) = active.best.take() {
4230 let cut = ready.cut;
4231 let value_count = ready.value_count;
4232 session.ready_current_view = Some(ready);
4233 json!({"state":"current_view_ready", "cut":cut,
4234 "value_count":value_count, "automatic_source":automatic,
4235 "attempted":attempted, "accepted_progress":false})
4236 } else if automatic {
4237 json!({"state":"current_view_unavailable", "attempted":attempted,
4238 "reason":active.reason.take().unwrap_or_else(|| "no current holder supplied an authenticated view".into()),
4239 "automatic_source":true, "accepted_progress":false})
4240 } else if active.reason.as_deref() == Some("holder has no current view") {
4241 json!({"state":"current_view_unavailable", "reason":active.reason.take().unwrap(),
4242 "automatic_source":false, "accepted_progress":false})
4243 } else {
4244 return Err(active
4245 .reason
4246 .take()
4247 .unwrap_or_else(|| "current-view request failed".into()));
4248 }
4249 } else if matches!(request, Request::StageCurrentView {}) {
4250 let (query, peer, reply, cut) = {
4251 let ready = session
4252 .ready_current_view
4253 .as_ref()
4254 .ok_or("no current view ready")?;
4255 (
4256 ready.query.clone(),
4257 ready.peer,
4258 ready.reply.clone(),
4259 ready.cut,
4260 )
4261 };
4262 let topics = BTreeSet::from([query.topic.clone()]);
4263 check_recovery_policy(
4264 session,
4265 peer,
4266 query.authority,
4267 query.workspace,
4268 query.epoch,
4269 query.policy_revision,
4270 &topics,
4271 )?;
4272 let owner = session
4273 .workspace
4274 .as_ref()
4275 .ok_or("session has no workspace")?;
4276 let key = session
4277 .storage_key
4278 .as_ref()
4279 .ok_or("session has no protected root key")?;
4280 let now = std::time::SystemTime::now()
4281 .duration_since(std::time::UNIX_EPOCH)
4282 .map_err(|_| "system clock is before Unix epoch")?
4283 .as_secs();
4284 let (mut inbox, pending, stale) = session
4285 .inbox
4286 .as_ref()
4287 .ok_or("object delivery not enabled")?
4288 .accept_current_view(owner, &query, &reply, now)
4289 .map_err(str::to_owned)?;
4290 if arachne_delivery::current::verify_wire_reply(owner, &query, &reply)
4291 .map_err(str::to_owned)?
4292 .is_some_and(|view| view.values.iter().any(|value| value.expires_at > now))
4293 {
4294 inbox = inbox
4295 .retain_current_view(owner, &query, &reply, now)
4296 .map_err(str::to_owned)?;
4297 }
4298 let publisher = session
4299 .publisher
4300 .clone()
4301 .unwrap_or(arachne_delivery::PublisherLog::new(
4302 owner.id(),
4303 owner.member().ok_or("member required")?.id(),
4304 owner.epoch(),
4305 ));
4306 let snapshot = seal_state(
4307 session.records.is_some(),
4308 owner,
4309 key,
4310 Some(&publisher),
4311 session.received.as_ref(),
4312 Some(&inbox),
4313 )?;
4314 let candidate = owner.provisional_copy().map_err(str::to_owned)?;
4315 session.staged_workspace = Some(StagedWorkspace {
4316 publisher: Some(publisher),
4317 received: session.received.clone(),
4318 inbox: Some(inbox),
4319 transition: WorkspaceTransition::CurrentView {
4320 cut,
4321 pending,
4322 stale,
4323 },
4324 workspace: candidate,
4325 snapshot: snapshot.clone(),
4326 });
4327 session.ready_current_view = None;
4328 json!({"workspace":owner.id(), "snapshot":snapshot,
4329 "state":"awaiting_current_view_save", "cut":cut,
4330 "pending":pending, "stale":stale, "durable":false,
4331 "accepted_progress":false})
4332 } else if matches!(request, Request::CancelCurrentView {}) {
4333 drop(session.current_view.take());
4334 session.ready_current_view = None;
4335 json!({"state":"current_view_cancelled", "accepted_progress":false})
4336 } else if let Request::DiscoverRecoveryCutoff {
4337 peer,
4338 revision,
4339 topics,
4340 } = request
4341 {
4342 if session.cutoff.is_some()
4343 || session.range.is_some()
4344 || session.ready_range.is_some()
4345 || session.direct_range.is_some()
4346 || session.ready_direct_range.is_some()
4347 || session.current_view.is_some()
4348 || session.ready_current_view.is_some()
4349 {
4350 return Err("recovery operation already pending".into());
4351 }
4352 let owner = session
4353 .workspace
4354 .as_ref()
4355 .ok_or("session has no workspace")?;
4356 if peer == session.node.id()
4357 || topics.is_empty()
4358 || topics.len() > arachne_delivery::MAX_TOPICS
4359 {
4360 return Err("invalid recovery peer or selection".into());
4361 }
4362 let count = topics.len();
4363 let topics = topics
4364 .into_iter()
4365 .map(Topic::new)
4366 .collect::<Result<BTreeSet<_>, _>>()
4367 .map_err(|e| e.to_string())?;
4368 if topics.len() != count {
4369 return Err("duplicate recovery topic".into());
4370 }
4371 let expected = owner
4372 .recovery_cutoff_request(peer, arachne_delivery::selection_digest(&topics), revision)
4373 .map_err(str::to_owned)?;
4374 let query = arachne_delivery::wire::CutoffQuery {
4375 workspace: expected.workspace,
4376 author: expected.author,
4377 epoch: expected.epoch,
4378 policy_revision: revision,
4379 topics,
4380 nonce: expected.nonce,
4381 };
4382 check_recovery_policy(
4383 session,
4384 peer,
4385 query.author,
4386 query.workspace,
4387 query.epoch,
4388 query.policy_revision,
4389 &query.topics,
4390 )?;
4391 let task = session.runtime.spawn(
4392 session
4393 .node
4394 .request_control(peer, &query.to_wire().map_err(str::to_owned)?),
4395 );
4396 session.cutoff = Some(PendingControl { query, peer, task });
4397 json!({"state":"recovery_cutoff_pending", "accepted_progress":false})
4398 } else if matches!(request, Request::PollRecoveryCutoff {}) {
4399 if !session
4400 .cutoff
4401 .as_ref()
4402 .is_some_and(|pending| pending.task.is_finished())
4403 {
4404 return Ok(Value::Null);
4405 }
4406 let mut pending = session.cutoff.take().unwrap();
4408 check_recovery_policy(
4409 session,
4410 pending.peer,
4411 pending.query.author,
4412 pending.query.workspace,
4413 pending.query.epoch,
4414 pending.query.policy_revision,
4415 &pending.query.topics,
4416 )?;
4417 let reply = session
4418 .runtime
4419 .block_on(&mut pending.task)
4420 .map_err(|_| "recovery cutoff task cancelled")?
4421 .map_err(|e| e.to_string())?;
4422 let owner = session
4423 .workspace
4424 .as_ref()
4425 .ok_or("session has no workspace")?;
4426 let head = if reply == arachne_delivery::wire::denied_reply() {
4427 None
4428 } else {
4429 Some(
4430 owner
4431 .verify_recovery_window(
4432 &pending.query.request().map_err(str::to_owned)?,
4433 &reply,
4434 )
4435 .map_err(str::to_owned)?,
4436 )
4437 };
4438 let query = &pending.query;
4439 match head {
4440 Some((after, head)) => {
4441 let accepted_through = session
4442 .inbox
4443 .as_ref()
4444 .map_or_else(
4445 || {
4446 session
4447 .received
4448 .as_ref()
4449 .and_then(|received| received.progress(query.author, &query.topics))
4450 },
4451 |inbox| Some(inbox.recovery_progress(query.author, &query.topics)),
4452 )
4453 .unwrap_or(0);
4454 json!({"state":"recovery_cutoff_observed", "workspace":owner.id(),
4455 "author":query.author, "peer":pending.peer, "epoch":owner.epoch(), "revision":query.policy_revision,
4456 "topics":query.topics.iter().map(|t|t.as_str()).collect::<Vec<_>>(),
4457 "head":head, "retained_after":after, "accepted_through":accepted_through,
4458 "accepted_progress":false})
4459 }
4460 None => json!({"state":"recovery_cutoff_denied", "accepted_progress":false}),
4461 }
4462 } else if let Request::FetchInvitationCheckpoint {
4463 peer,
4464 mut peers,
4465 invitation,
4466 } = request
4467 {
4468 if let Some(peer) = peer {
4469 peers.insert(0, peer);
4470 }
4471 return session.runtime.block_on(request_invitation_checkpoint(
4472 session.node.control_client(),
4473 session.node.id(),
4474 invitation,
4475 peers,
4476 ));
4477 } else if let Request::JoinViaPeer { peer } = request {
4478 let pending = session
4479 .pending_join
4480 .as_ref()
4481 .ok_or("session has no pending join")?;
4482 let request = pending.admission_request().map_err(str::to_owned)?;
4483 let name = pending.member().display_name().as_bytes();
4484 let mut packet = b"DFJA\x02".to_vec();
4485 packet.extend((request.len() as u32).to_be_bytes());
4486 packet.extend((name.len() as u16).to_be_bytes());
4487 packet.extend(request);
4488 packet.extend(name);
4489 packet.extend(pending.admission_checkpoint().map_err(str::to_owned)?);
4490 let outcome = session
4491 .runtime
4492 .block_on(session.node.request_control(peer, &packet));
4493 let reply = match outcome {
4494 Ok(reply) => reply,
4495 Err(arachne_node::Error::ControlNotSent(_) | arachne_node::Error::MissingPeer) => {
4496 return Ok(json!({"state":"admission_not_sent","peer":peer}));
4497 }
4498 Err(arachne_node::Error::Timeout(_) | arachne_node::Error::Transport(_)) => {
4501 return Ok(json!({"state":admission_state::WAITING,"peer":peer}));
4502 }
4503 Err(error) => return Err(error.to_string()),
4504 };
4505 let mut page_bytes = vec![reply.len()];
4509 let mut reply: Value =
4510 serde_json::from_slice(&reply).map_err(|_| "invalid admission reply")?;
4511 if reply
4512 .get("history_complete")
4513 .is_some_and(|complete| !complete.as_bool().unwrap_or(false))
4514 {
4515 let mut commits = reply["commits"]
4516 .as_array()
4517 .cloned()
4518 .ok_or("admission history page missing commits")?;
4519 let mut offset = reply["history_next"]
4520 .as_u64()
4521 .ok_or("admission history page missing next offset")?
4522 as usize;
4523 let mut page_count = 0;
4524 let mut total_bytes: usize = page_bytes.iter().sum();
4529 while !reply["history_complete"].as_bool().unwrap_or(false) {
4530 page_count += 1;
4531 if page_count > arachne_security::MAX_JOIN_HISTORY_STEPS {
4535 return Err("admission history page count exceeds bounds".into());
4536 }
4537 let page = admission_history_page_packet(
4538 request,
4539 pending.admission_checkpoint().map_err(str::to_owned)?,
4540 offset,
4541 )?;
4542 let page = session
4543 .runtime
4544 .block_on(session.node.request_control(peer, &page))
4545 .map_err(|error| error.to_string())?;
4546 total_bytes = total_bytes.saturating_add(page.len());
4547 if total_bytes > arachne_security::MAX_JOIN_HISTORY_BYTES {
4548 return Err("admission history exceeds transport bounds".into());
4549 }
4550 page_bytes.push(page.len());
4551 let page: Value =
4552 serde_json::from_slice(&page).map_err(|_| "invalid admission history page")?;
4553 if page.get("history_page").and_then(Value::as_bool) != Some(true) {
4558 return Err("admission history page was not accepted".into());
4559 }
4560 if page["history_offset"].as_u64() != Some(offset as u64) {
4561 return Err("admission history page offset mismatch".into());
4562 }
4563 let page_commits = page["commits"]
4564 .as_array()
4565 .ok_or("admission history page missing commits")?;
4566 let next = page["history_next"]
4567 .as_u64()
4568 .ok_or("admission history page missing next offset")?
4569 as usize;
4570 if page_commits.is_empty() || next <= offset {
4571 return Err("admission history page made no progress".into());
4572 }
4573 if commits.len() + page_commits.len() > arachne_security::MAX_JOIN_HISTORY_STEPS {
4574 return Err("admission history exceeds step bounds".into());
4575 }
4576 commits.extend(page_commits.iter().cloned());
4577 offset = next;
4578 reply = page;
4579 }
4580 reply["commits"] = Value::Array(commits);
4581 reply["history_complete"] = Value::Bool(true);
4582 }
4583 let total = reply["commits"].as_array().map_or(0, Vec::len);
4588 if total > arachne_security::HISTORY_CHUNK_STEPS {
4589 let trailing = match total % arachne_security::HISTORY_CHUNK_STEPS {
4590 0 => arachne_security::HISTORY_CHUNK_STEPS,
4591 remainder => remainder,
4592 };
4593 let split = total - trailing;
4594 let commits = reply["commits"].as_array().unwrap();
4595 let prefix = commits[..split].to_vec();
4596 let carried = Value::Array(commits[split..].to_vec());
4597 reply["commits"] = carried;
4598 reply["history_verified_prefix"] = json!(split);
4599 session.join_history_prefix = prefix;
4600 } else {
4601 session.join_history_prefix.clear();
4602 }
4603 if reply.get("commits").is_some() {
4604 reply["history_page_bytes"] = json!(page_bytes);
4607 }
4608 reply
4609 } else if let Request::ListAdmissionApprovals { after, limit } = request {
4610 list_pending_approvals(session, after, limit)?
4611 } else if let Request::AcknowledgeAdmissionApproval { attempt_id } = request {
4612 let pending = session
4613 .pending_approvals
4614 .get_mut(&attempt_id)
4615 .ok_or("admission approval is no longer pending")?;
4616 pending.acknowledged = true;
4617 json!({"state":admission_state::APPROVAL_PENDING,"attempt_id":attempt_id,"acknowledged":true})
4618 } else if let Request::PollAdmission { profile } = request {
4619 reap_admission_pushes(session);
4620 if let Some(incoming) = session
4624 .node
4625 .poll_control_first(|payload| payload.starts_with(b"DFMS"), RANGE_SCAN_DEPTH)
4626 {
4627 let waited_ms = incoming.waited().as_millis() as u64;
4628 let reply = membership::range_reply(
4629 session.workspace.as_deref(),
4630 incoming.peer(),
4631 incoming.payload(),
4632 );
4633 tracing::info!(target: "data_fabric_transport", waited_ms, "RANGE_REQUEST_SERVED");
4634 let _ = incoming.respond(reply);
4635 return Ok(json!({"state":"membership_replied", "remote_receipt":false}));
4636 }
4637 membership::send_queued_profiles(session);
4640 let admission_busy = admission_busy(session);
4641 if !admission_busy && let Some(staged) = membership::stage_gossiped_step(session)? {
4644 return Ok(staged);
4645 }
4646 if !admission_busy
4651 && should_stage_queued_admission(session)
4652 && let Some(staged) = stage_queued_admission(session)?
4653 {
4654 return Ok(staged);
4655 }
4656 let incoming = if admission_busy || !session.admission_queue.is_empty() {
4659 session
4660 .node
4661 .poll_control_matching(admission_packet_candidate)
4662 } else {
4663 session.node.poll_control()
4664 };
4665 if incoming.is_none()
4666 && !admission_busy
4667 && let Some(staged) = stage_queued_admission(session)?
4668 {
4669 return Ok(staged);
4670 }
4671 let incoming = incoming.or_else(|| {
4672 if admission_busy {
4673 None
4674 } else {
4675 session.node.poll_control()
4676 }
4677 });
4678 let Some(incoming) = incoming else {
4679 return Ok(membership::take_answered(session).unwrap_or(Value::Null));
4682 };
4683 if incoming.payload() == NEARBY_IDENTITY {
4684 incoming
4685 .respond(
4686 session
4687 .nearby_identity
4688 .as_deref()
4689 .unwrap_or("Unnamed Arachne device")
4690 .as_bytes()
4691 .to_vec(),
4692 )
4693 .map_err(|error| error.to_string())?;
4694 return Ok(json!({"state":"nearby_identity_replied"}));
4695 }
4696 if incoming.payload() == NEARBY_WORKSPACE {
4697 incoming
4698 .respond(nearby_workspace_reply(&session.nearby_workspaces))
4699 .map_err(|error| error.to_string())?;
4700 return Ok(json!({"state":"nearby_workspace_replied"}));
4701 }
4702 if incoming.payload().starts_with(NEARBY_INVITATION) {
4703 let payload = incoming.payload();
4704 if payload.len() < 7 {
4705 let _ = incoming.respond(vec![0]);
4706 return Ok(json!({"state":"nearby_invitation_rejected"}));
4707 }
4708 let length = u16::from_be_bytes(payload[5..7].try_into().unwrap()) as usize;
4709 if length == 0 || length > 2048 || payload.len() != 7 + length {
4710 let _ = incoming.respond(vec![0]);
4711 return Ok(json!({"state":"nearby_invitation_rejected"}));
4712 }
4713 let peer = incoming.peer();
4714 let invitation = payload[7..].to_vec();
4715 incoming
4716 .respond(vec![1])
4717 .map_err(|error| error.to_string())?;
4718 return Ok(
4719 json!({"state":"nearby_invitation_received","peer":peer,"invitation":invitation}),
4720 );
4721 }
4722 if incoming.payload().starts_with(b"DFPR") {
4723 let peer = incoming.peer();
4724 let result = presence::receive(session, peer, incoming.payload());
4725 if result.is_ok()
4726 && let Some(address) = incoming.remote_address()
4727 {
4728 session
4729 .runtime
4730 .block_on(session.node.remember_observed(peer, address));
4731 }
4732 let accepted = result.is_ok();
4733 let _ = incoming.respond(result.unwrap_or_else(|_| vec![0]));
4734 return Ok(json!({"state":"presence_replied","accepted":accepted}));
4735 }
4736 if incoming.payload().starts_with(b"DFLV") {
4737 let value =
4738 match membership::receive_leave(session, incoming.peer(), incoming.payload()) {
4739 Ok(value) => value,
4740 Err(_) => {
4741 let _ = incoming.respond(b"{\"state\":\"leave_denied\"}".to_vec());
4742 return Ok(json!({"state":"membership_replied"}));
4743 }
4744 };
4745 session.inbound_admission = Some(incoming);
4746 return Ok(value);
4747 }
4748 if incoming.payload().starts_with(b"DFMO") {
4749 let offered = membership::receive_offer(session, incoming.payload());
4750 return match offered {
4751 Ok(value) => {
4752 session.inbound_admission = Some(incoming);
4753 Ok(value)
4754 }
4755 Err(_) => {
4756 let _ = incoming.respond(vec![0]);
4757 Ok(json!({"state":"membership_replied"}))
4758 }
4759 };
4760 }
4761 if incoming
4762 .payload()
4763 .starts_with(INVITATION_CHECKPOINT_REQUEST)
4764 {
4765 let reply = invitation_checkpoint_reply(session, incoming.peer(), incoming.payload());
4766 let accepted = reply.is_ok();
4767 let _ = incoming.respond(reply.unwrap_or_default());
4768 return Ok(json!({"state":"invitation_checkpoint_replied","accepted":accepted}));
4769 }
4770 if incoming.payload().starts_with(b"DFMS") {
4771 let reply = membership::range_reply(
4772 session.workspace.as_deref(),
4773 incoming.peer(),
4774 incoming.payload(),
4775 );
4776 let _ = incoming.respond(reply);
4777 return Ok(json!({"state":"membership_replied", "remote_receipt":false}));
4778 }
4779 if incoming
4780 .payload()
4781 .starts_with(membership::PROFILE_QUERY_PREFIX)
4782 {
4783 let reply = membership::profile_page_reply(
4784 session.workspace.as_deref(),
4785 &membership::lock_profiles(&session.profiles),
4786 incoming.peer(),
4787 incoming.payload(),
4788 );
4789 let _ = incoming.respond(reply);
4790 return Ok(json!({"state":"membership_replied", "remote_receipt":false}));
4791 }
4792 if incoming.payload().starts_with(b"DFMQ") {
4793 let peer = incoming.peer();
4794 let reply = membership::reply_with_profiles(session, peer, incoming.payload());
4795 let current_peer = reply["state"] != "membership_denied"
4796 && session
4797 .workspace
4798 .as_ref()
4799 .is_some_and(|owner| owner.member_id_for_endpoint(peer).is_ok());
4800 incoming
4801 .respond(membership::encode_reply(&reply)?)
4802 .map_err(|e| e.to_string())?;
4803 let mut event = json!({"state":"membership_replied", "remote_receipt":false});
4804 if current_peer {
4805 event["peer"] = json!(peer);
4806 }
4807 return Ok(event);
4808 }
4809 if incoming.payload().starts_with(b"DFRQ")
4812 || incoming.payload().starts_with(b"DFHQ")
4813 || incoming.payload().starts_with(b"DFCQ")
4814 || incoming.payload().starts_with(b"DFVQ")
4815 || incoming.payload().starts_with(b"DFDQ")
4816 {
4817 let reply = match session.workspace.as_ref() {
4818 Some(owner) => {
4819 let now = std::time::SystemTime::now()
4820 .duration_since(std::time::UNIX_EPOCH)
4821 .map_err(|_| "system clock is before Unix epoch")?
4822 .as_secs();
4823 session
4824 .runtime
4825 .block_on(session.node.with_routing_policy(|policy| {
4826 if incoming.payload().starts_with(b"DFDQ") {
4827 match (
4828 session.inbox.as_ref(),
4829 arachne_delivery::wire::DirectRangeQuery::from_wire(
4830 incoming.payload(),
4831 ),
4832 ) {
4833 (Some(inbox), Ok(query)) => inbox.serve_direct_range(
4834 owner,
4835 policy,
4836 incoming.peer(),
4837 &query,
4838 ),
4839 _ => Ok(arachne_delivery::wire::unavailable_direct_reply()),
4840 }
4841 } else if incoming.payload().starts_with(b"DFVQ") {
4842 match (
4843 session.inbox.as_ref(),
4844 arachne_delivery::current::CurrentViewQuery::from_wire(
4845 incoming.payload(),
4846 ),
4847 ) {
4848 (Some(inbox), Ok(query)) => inbox.serve_current(
4849 owner,
4850 policy,
4851 incoming.peer(),
4852 &query,
4853 now,
4854 ),
4855 _ => Ok(arachne_delivery::current::CurrentView::denied_wire()),
4856 }
4857 } else if incoming.payload().starts_with(b"DFCQ") {
4858 match (
4859 session.publisher.as_ref(),
4860 arachne_delivery::wire::CutoffQuery::from_wire(
4861 incoming.payload(),
4862 ),
4863 ) {
4864 (Some(log), Ok(query)) => arachne_delivery::wire::serve_cutoff(
4865 log,
4866 owner,
4867 policy,
4868 incoming.peer(),
4869 &query,
4870 ),
4871 _ => Ok(arachne_delivery::wire::denied_reply()),
4872 }
4873 } else if incoming.payload().starts_with(b"DFHQ") {
4874 let Ok(query) =
4875 arachne_delivery::wire::AvailableRangeQuery::from_wire(
4876 incoming.payload(),
4877 )
4878 else {
4879 return Ok(
4880 arachne_delivery::wire::unavailable_available_reply(),
4881 );
4882 };
4883 if owner.member().map(|member| member.id()) == Some(query.author) {
4884 match session.publisher.as_ref() {
4885 Some(log) => arachne_delivery::wire::serve_available_range(
4886 log,
4887 owner,
4888 policy,
4889 incoming.peer(),
4890 &query,
4891 ),
4892 None => Ok(
4893 arachne_delivery::wire::unavailable_available_reply(),
4894 ),
4895 }
4896 } else {
4897 match session.inbox.as_ref() {
4898 Some(inbox) => inbox.serve_available_range(
4899 owner,
4900 policy,
4901 incoming.peer(),
4902 &query,
4903 now,
4904 ),
4905 None => Ok(
4906 arachne_delivery::wire::unavailable_available_reply(),
4907 ),
4908 }
4909 }
4910 } else {
4911 let Ok(query) =
4912 arachne_delivery::RangeQuery::from_wire(incoming.payload())
4913 else {
4914 return Ok(arachne_delivery::wire::denied_reply());
4915 };
4916 if owner.member().map(|member| member.id()) == Some(query.author) {
4917 match session.publisher.as_ref() {
4918 Some(log) => arachne_delivery::wire::serve_range(
4919 log,
4920 owner,
4921 policy,
4922 incoming.peer(),
4923 &query,
4924 ),
4925 None => Ok(arachne_delivery::wire::denied_reply()),
4926 }
4927 } else {
4928 match session.inbox.as_ref() {
4929 Some(inbox) => inbox.serve_range(
4930 owner,
4931 policy,
4932 incoming.peer(),
4933 &query,
4934 now,
4935 ),
4936 None => Ok(arachne_delivery::wire::denied_reply()),
4937 }
4938 }
4939 }
4940 }))
4941 .map_err(str::to_owned)?
4942 }
4943 None if incoming.payload().starts_with(b"DFVQ") => {
4944 arachne_delivery::current::CurrentView::denied_wire()
4945 }
4946 None if incoming.payload().starts_with(b"DFHQ") => {
4947 arachne_delivery::wire::unavailable_available_reply()
4948 }
4949 None if incoming.payload().starts_with(b"DFDQ") => {
4950 arachne_delivery::wire::unavailable_direct_reply()
4951 }
4952 None => arachne_delivery::wire::denied_reply(),
4953 };
4954 let state = if incoming.payload().starts_with(b"DFVQ") {
4955 "current_view_replied"
4956 } else if incoming.payload().starts_with(b"DFDQ") {
4957 "direct_recovery_replied"
4958 } else {
4959 "recovery_replied"
4960 };
4961 incoming.respond(reply).map_err(|e| e.to_string())?;
4962 return Ok(json!({"state":state, "remote_receipt":false}));
4963 }
4964 if admission_packet_candidate(incoming.payload()) {
4965 let mut value = queue_admission(session, incoming)?;
4966 if !profile && let Some(object) = value.as_object_mut() {
4970 object.remove("intake");
4971 }
4972 return Ok(value);
4973 }
4974 let result: Result<Value, String> = (|| {
4975 let workspace = session
4976 .workspace
4977 .as_ref()
4978 .ok_or("session has no workspace")?;
4979 let (request, checkpoint) = admission_parts(incoming.payload())?;
4980 if let Some(checkpoint) = checkpoint {
4981 workspace
4982 .membership_history(incoming.peer(), request, checkpoint)
4983 .map_err(str::to_owned)?;
4984 }
4985 let retained = workspace
4986 .retained_admission(incoming.peer(), request)
4987 .map_err(str::to_owned)?
4988 .is_some();
4989 let value = if retained {
4990 json!({"workspace":workspace.id(),"state":"reply_ready"})
4991 } else {
4992 stage_admission(session, incoming.peer(), request, checkpoint, None)?
4993 };
4994 Ok(value)
4995 })();
4996 let value = match result {
4997 Ok(value) => value,
4998 Err(error) => {
4999 let feedback = admission_feedback(&error);
5000 let reason = feedback["reason"]
5001 .as_str()
5002 .ok_or("admission feedback has no reason")?;
5003 let approval = if reason == "approval_required"
5006 || reason == "automatic_approval_required"
5007 {
5008 let (request, _, display_name) = admission_packet(incoming.payload())?;
5009 Some((
5010 incoming.peer(),
5011 request.to_vec(),
5012 display_name.map(str::to_owned),
5013 ))
5014 } else {
5015 None
5016 };
5017 let _ = incoming.respond(
5018 serde_json::to_vec(&feedback)
5019 .map_err(|_| "admission feedback encoding failed")?,
5020 );
5021 if let Some((endpoint, request, display_name)) = approval {
5022 return Ok(
5023 json!({"state":"approval_requested","endpoint":endpoint,"request":request,"display_name":display_name,"automatic":reason == "automatic_approval_required"}),
5024 );
5025 }
5026 let mut response = json!({
5027 "state": admission_state::REPLIED,
5028 "accepted": false,
5029 "reason": reason,
5030 });
5031 if let Some(recovery) = feedback.get("recovery") {
5032 response["recovery"] = recovery.clone();
5033 }
5034 return Ok(response);
5035 }
5036 };
5037 session.inbound_admission = Some(incoming);
5038 value
5039 } else if matches!(request, Request::SendAdmissionReply {}) {
5040 send_inbound_admission_reply(session)?
5041 } else if let Request::StageJoin { commits, welcome } = request {
5042 if commits.is_empty() || commits.len() > arachne_security::HISTORY_CHUNK_STEPS {
5043 return Err("join history exceeds step bounds".into());
5044 }
5045 let pending = session
5046 .pending_join
5047 .as_ref()
5048 .ok_or("session has no pending join")?;
5049 let key = session
5050 .storage_key
5051 .as_ref()
5052 .ok_or("session has no protected root key")?;
5053 let mut proof = pending.join_proof().map_err(str::to_owned)?;
5054 for value in &session.join_history_prefix {
5059 let step: JoinStep =
5060 serde_json::from_value(value.clone()).map_err(|error| error.to_string())?;
5061 proof
5062 .apply_transition(&step.authorization()?, &step.commit)
5063 .map_err(str::to_owned)?;
5064 }
5065 for step in commits {
5066 proof
5067 .apply_transition(&step.authorization()?, &step.commit)
5068 .map_err(str::to_owned)?;
5069 }
5070 let workspace = pending
5071 .prepare_workspace(&proof, &welcome)
5072 .map_err(str::to_owned)?;
5073 let snapshot = seal_state(session.records.is_some(), &workspace, key, None, None, None)?;
5074 let value = json!({"workspace":workspace.id(), "workspace_name":workspace.workspace_name().map_err(str::to_owned)?, "snapshot":snapshot, "state":"awaiting_join_save", "durable":false});
5075 session.staged_workspace = Some(StagedWorkspace {
5076 publisher: None,
5077 received: None,
5078 inbox: None,
5079 transition: WorkspaceTransition::Join,
5080 workspace,
5081 snapshot,
5082 });
5083 transition_activity(session, WorkspacePhase::Synchronizing, None)?;
5084 let mut value = value;
5085 value["activity"] = activity_value(session);
5086 value
5087 } else if let Request::MemberRoster { profiles } = request {
5088 membership::roster(session, &profiles)?
5089 } else if matches!(request, Request::UseServiceProfile {}) {
5090 membership::lock_profiles(&session.profiles).service = true;
5091 json!({"state":"service_profile"})
5092 } else if matches!(
5093 request,
5094 Request::StageWorkspaceName { .. }
5095 | Request::StageWorkspaceNameUpdate { .. }
5096 | Request::StageWorkspaceNameCheckpoint { .. }
5097 ) {
5098 let owner = session
5099 .workspace
5100 .as_ref()
5101 .ok_or("session has no workspace")?;
5102 let (workspace, missing) = match request {
5103 Request::StageWorkspaceName { workspace_name } => owner
5104 .prepare_workspace_name(&workspace_name)
5105 .map(|p| (p.workspace, None)),
5106 Request::StageWorkspaceNameUpdate { name_record } => owner
5107 .prepare_workspace_name_update(&name_record)
5108 .map(|workspace| (workspace, None)),
5109 Request::StageWorkspaceNameCheckpoint { name_checkpoint } => owner
5110 .prepare_workspace_name_checkpoint(&name_checkpoint)
5111 .map(|prepared| (prepared.workspace, Some(prepared.missing))),
5112 _ => unreachable!(),
5113 }
5114 .map_err(str::to_owned)?;
5115 let snapshot = seal_state(
5116 session.records.is_some(),
5117 &workspace,
5118 session
5119 .storage_key
5120 .as_ref()
5121 .ok_or("session has no protected root key")?,
5122 session.publisher.as_ref(),
5123 session.received.as_ref(),
5124 session.inbox.as_ref(),
5125 )?;
5126 let mut value = json!({"workspace":workspace.id(),"workspace_name":workspace.workspace_name().map_err(str::to_owned)?,
5127 "snapshot":snapshot,"state":"awaiting_save","durable":false});
5128 if let Some(missing) = missing {
5129 value["name_history_missing_added"] = json!(missing);
5130 value["name_history_missing"] = json!(
5131 workspace
5132 .workspace_name_missing_history()
5133 .map_err(str::to_owned)?
5134 );
5135 }
5136 session.staged_workspace = Some(StagedWorkspace {
5137 publisher: session.publisher.clone(),
5138 received: session.received.clone(),
5139 inbox: session.inbox.clone(),
5140 transition: WorkspaceTransition::WorkspaceName,
5141 workspace,
5142 snapshot,
5143 });
5144 value
5145 } else if let Request::StageInvitation {
5146 expires_at,
5147 personal,
5148 automatic,
5149 request_access,
5150 } = request
5151 {
5152 check_epoch_transition(session)?;
5153 let owner = session
5154 .workspace
5155 .as_ref()
5156 .ok_or("session has no workspace")?;
5157 if request_access && (!personal || automatic) {
5158 return Err("invalid request-access mode".into());
5159 }
5160 let (prepared, invitation, checkpoint) = if request_access {
5161 owner.prepare_request_invitation(expires_at)
5162 } else {
5163 owner.prepare_invitation(expires_at, personal, automatic)
5164 }
5165 .map_err(str::to_owned)?;
5166 let action = prepared.action;
5167 let commit = prepared.commit.clone();
5168 let value = membership::stage_prepared(session, prepared)?;
5169 session.staged_workspace.as_mut().unwrap().transition =
5170 WorkspaceTransition::Invitation(Box::new(invitation), checkpoint, action, commit);
5171 value
5172 } else if let Request::StageInvitationApproval {
5173 request,
5174 attempt_id,
5175 } = request
5176 {
5177 check_epoch_transition(session)?;
5178 let pending_id = pending_approval_id(session, &request, attempt_id)?;
5179 let prepared = session
5180 .workspace
5181 .as_ref()
5182 .ok_or("session has no workspace")?
5183 .prepare_invitation_approval(&request)
5184 .map_err(str::to_owned)?;
5185 let value = membership::stage_prepared(session, prepared)?;
5186 session.staged_approval_id = pending_id;
5187 value
5188 } else if let Request::StageInvitationDecline {
5189 request,
5190 attempt_id,
5191 } = request
5192 {
5193 check_epoch_transition(session)?;
5194 let pending_id = pending_approval_id(session, &request, attempt_id)?;
5195 let prepared = session
5196 .workspace
5197 .as_ref()
5198 .ok_or("session has no workspace")?
5199 .prepare_invitation_decline(&request)
5200 .map_err(str::to_owned)?;
5201 let value = membership::stage_prepared(session, prepared)?;
5202 session.staged_approval_id = pending_id;
5203 value
5204 } else if matches!(request, Request::InvitationControls {}) {
5205 let owner = session
5206 .workspace
5207 .as_ref()
5208 .ok_or("session has no workspace")?;
5209 let (legacy, controls) = owner.invitation_controls().map_err(str::to_owned)?;
5210 json!({"legacy_enabled":legacy,"invitations":controls.iter().filter(|c| !c.is_request_decision(&controls)).enumerate().map(|(i,c)| json!({"number":i+1,"key":c.key,"expires_at":c.expires_at,"enabled":c.enabled,"personal":c.personal,"automatic":c.automatic(),"request_access":c.request_access(),"approved":c.approved()})).collect::<Vec<_>>()})
5211 } else if let Request::StageManagement { action } = request {
5212 membership::stage_management(session, action.action()?)?
5213 } else if let Request::LeaveViaPeer { peer } = request {
5214 membership::leave_via_peer(session, peer)?
5215 } else if matches!(request, Request::StageSoloLeave {}) {
5216 check_epoch_transition(session)?;
5217 let ended = session
5218 .workspace
5219 .as_ref()
5220 .ok_or("session has no workspace")?
5221 .prepare_solo_leave()
5222 .map_err(str::to_owned)?;
5223 membership::stage_removal(session, ended)?
5224 } else if let Request::StageAdmissionUpdate { step } = request {
5225 membership::stage_update(session, step)?
5226 } else if let Request::StageAdmission {
5227 authenticated_endpoint,
5228 request,
5229 } = request
5230 {
5231 stage_admission(session, authenticated_endpoint, &request, None, None)?
5232 } else if matches!(
5233 request,
5234 Request::AdoptAdmission { .. }
5235 | Request::AdoptJoin { .. }
5236 | Request::AdoptPublication { .. }
5237 | Request::AdoptReception { .. }
5238 | Request::AdoptRecovery { .. }
5239 | Request::AdoptCurrentView { .. }
5240 ) {
5241 let staged = session
5242 .staged_workspace
5243 .as_ref()
5244 .ok_or("session has no workspace candidate")?;
5245 let valid_phase = matches!(
5246 (&request, &staged.transition),
5247 (
5248 Request::AdoptAdmission { .. },
5249 WorkspaceTransition::Admission
5250 | WorkspaceTransition::Management(_, _)
5251 | WorkspaceTransition::WorkspaceName
5252 | WorkspaceTransition::Invitation(..)
5253 ) | (Request::AdoptJoin { .. }, WorkspaceTransition::Join)
5254 | (
5255 Request::AdoptRecovery { .. },
5256 WorkspaceTransition::Recovery(_)
5257 | WorkspaceTransition::InboxRecovery { .. }
5258 | WorkspaceTransition::DirectMiss { .. }
5259 )
5260 | (
5261 Request::AdoptCurrentView { .. },
5262 WorkspaceTransition::CurrentView { .. }
5263 )
5264 | (
5265 Request::AdoptPublication { .. },
5266 WorkspaceTransition::Publication(_)
5267 | WorkspaceTransition::RoutedPublication(..)
5268 )
5269 | (
5270 Request::AdoptReception { .. },
5271 WorkspaceTransition::Reception(_)
5272 | WorkspaceTransition::RoutedReception(..)
5273 | WorkspaceTransition::Inbox
5274 | WorkspaceTransition::InboxRejected
5275 )
5276 );
5277 if !valid_phase {
5278 return Err("wrong adoption lifecycle phase".into());
5279 }
5280 let snapshot = match request {
5281 Request::AdoptAdmission { snapshot }
5282 | Request::AdoptJoin { snapshot }
5283 | Request::AdoptPublication { snapshot }
5284 | Request::AdoptReception { snapshot }
5285 | Request::AdoptRecovery { snapshot }
5286 | Request::AdoptCurrentView { snapshot } => snapshot,
5287 _ => unreachable!(),
5288 };
5289 if snapshot != staged.snapshot {
5290 return Err("workspace snapshot does not match candidate".into());
5291 }
5292 if let Some(store) = &session.records {
5293 store.require_committed(&snapshot)?;
5294 }
5295 let staged = session.staged_workspace.take().unwrap();
5296 let joined = matches!(&staged.transition, WorkspaceTransition::Join);
5297 let mut value = json!({"workspace":staged.workspace.id(), "workspace_name":staged.workspace.workspace_name().map_err(str::to_owned)?, "epoch":staged.workspace.epoch(),
5298 "workspace_name_missing_history":staged.workspace.workspace_name_missing_history().map_err(str::to_owned)?,
5299 "members":staged.workspace.member_count(), "durable":session.records.is_some()});
5300 session.publisher = staged.publisher;
5301 session.received = staged.received;
5302 session.inbox = staged.inbox;
5303 if matches!(&staged.transition, WorkspaceTransition::Join) {
5304 transition_activity(session, WorkspacePhase::Synchronizing, None)?;
5305 }
5306 commit_workspace(session, staged.workspace);
5307 let staged_approval_id = session.staged_approval_id;
5308 let received = std::mem::take(&mut session.staged_step_received);
5311 let committed_here =
5312 matches!(staged.transition, WorkspaceTransition::Admission) && !received;
5313 match staged.transition {
5314 WorkspaceTransition::Inbox => value["state"] = json!("inbox_adopted"),
5315 WorkspaceTransition::InboxRejected => value["state"] = json!("inbox_rejection_adopted"),
5316 WorkspaceTransition::InboxRecovery { count } => {
5317 value["state"] = json!("recovery_adopted");
5318 value["publication_count"] = json!(count);
5319 }
5320 WorkspaceTransition::DirectMiss { missing } => {
5321 value["state"] = json!("direct_miss_adopted");
5322 value["missing_count"] = json!(missing);
5323 }
5324 WorkspaceTransition::CurrentView {
5325 cut,
5326 pending,
5327 stale,
5328 } => {
5329 value["state"] = json!("current_view_adopted");
5330 value["cut"] = json!(cut);
5331 value["pending"] = json!(pending);
5332 value["stale"] = json!(stale);
5333 }
5334 WorkspaceTransition::Recovery(publications) => {
5335 value["state"] = json!("recovery_adopted");
5336 value["publication_count"] = json!(publications.len());
5337 session.recovered = publications.into();
5340 }
5341 WorkspaceTransition::RoutedPublication(
5342 context,
5343 delivery,
5344 packet,
5345 endpoints,
5346 recipients,
5347 ) => {
5348 let sent = session.runtime.block_on(async {
5350 tokio::time::timeout(Duration::from_secs(10), async {
5351 if recipients.is_empty() {
5352 session
5353 .node
5354 .publish_with_class(
5355 context.workspace,
5356 context.revision,
5357 context.topic,
5358 delivery,
5359 packet,
5360 )
5361 .await
5362 } else {
5363 session
5364 .node
5365 .publish_to_with_class(
5366 context.workspace,
5367 context.revision,
5368 context.topic,
5369 endpoints,
5370 recipients.clone(),
5371 delivery,
5372 packet,
5373 )
5374 .await
5375 }
5376 })
5377 .await
5378 });
5379 value["id"] = json!(context.id);
5380 value["sequence"] = json!(context.sequence.map(|n| n.get()));
5381 if !recipients.is_empty() {
5382 value["recipients"] = json!(recipients);
5383 }
5384 match sent {
5385 Ok(Ok(result)) => value["admission"] = report(result),
5386 Ok(Err(error)) => value["network_error"] = json!(error.to_string()),
5387 Err(_) => {
5388 value["network_error"] =
5389 json!("publication deadline exceeded; outcome may be partial")
5390 }
5391 }
5392 }
5393 WorkspaceTransition::RoutedReception(context, message, recipients) => {
5394 value["revision"] = json!(context.revision);
5395 value["topic"] = json!(context.topic.as_str());
5396 value["id"] = json!(context.id);
5397 value["sequence"] = json!(context.sequence.map(|n| n.get()));
5398 value["payload"] = json!(message.payload);
5399 value["member"] = json!(message.member);
5400 value["endpoint"] = json!(message.endpoint);
5401 if !recipients.is_empty() {
5402 value["recipients"] = json!(recipients);
5403 }
5404 }
5405 WorkspaceTransition::Management(action, commit) => {
5406 value["step"] = membership::step_json(
5407 &arachne_security::MembershipAuthorization::Management(action),
5408 &commit,
5409 );
5410 }
5411 WorkspaceTransition::Invitation(invitation, checkpoint, action, commit) => {
5412 value["issued_invitation"] = invitation_envelope(session, &invitation, checkpoint)?;
5413 let mut step = membership::step_json(
5414 &arachne_security::MembershipAuthorization::Management(action),
5415 &commit,
5416 );
5417 step["invitation_checkpoint"] = json!({
5418 "grant": invitation.public_grant(),
5419 "checkpoint": value["issued_invitation"]["checkpoint"],
5420 });
5421 value["step"] = step;
5422 }
5423 WorkspaceTransition::Admission => {
5424 let admitted = std::mem::take(&mut session.queued_admission_in_flight);
5425 let mut delivered = 0usize;
5426 let mut pushed = 0usize;
5427 if let Some(workspace) = session.workspace.clone() {
5428 for attempt in &admitted {
5429 let held = session.admission_waiters.take(&attempt.id());
5430 let checkpoint = held.as_ref().and_then(|(_, checkpoint)| checkpoint.as_deref());
5431 let Ok(reply) = admission_reply_page(
5432 &workspace,
5433 attempt.endpoint(),
5434 attempt.request(),
5435 checkpoint,
5436 0,
5437 ) else {
5438 continue;
5439 };
5440 let (delivered_here, route) = match held {
5441 Some((exchange, _)) => {
5442 let route = exchange.remote_address();
5443 let delivered = !exchange.expired()
5444 && exchange.respond(reply.clone()).is_ok();
5445 (delivered, route)
5446 }
5447 None => (false, None),
5448 };
5449 if delivered_here {
5450 delivered += 1;
5451 } else if queue_admission_push(session, attempt, &reply, route) {
5452 pushed += 1;
5453 }
5454 }
5455 }
5456 value["results_delivered"] = json!(delivered);
5457 value["results_pushed"] = json!(pushed);
5458 }
5459 WorkspaceTransition::WorkspaceName => {}
5460 WorkspaceTransition::Join => {
5461 session.pending_join = None;
5462 session.join_lifecycle = None;
5463 session.join_history_prefix.clear();
5464 transition_activity(session, WorkspacePhase::Active, None)?;
5465 }
5466 WorkspaceTransition::Publication(ciphertext) => value["ciphertext"] = json!(ciphertext),
5467 WorkspaceTransition::Reception(message) => {
5468 value["payload"] = json!(message.payload);
5469 value["member"] = json!(message.member);
5470 value["endpoint"] = json!(message.endpoint);
5471 }
5472 }
5473 if joined && session.inbound_admission.is_some() {
5474 let reply = send_inbound_admission_reply(session)?;
5475 value["reply_queued"] = json!(reply["queued"]);
5476 }
5477 if let Some(id) = staged_approval_id {
5478 session.pending_approvals.remove(&id);
5479 session.staged_approval_id = None;
5480 }
5481 let reached_head = received
5484 && session.workspace.as_ref().is_some_and(|owner| {
5485 !session.gossip_steps_ahead.contains_key(&owner.epoch())
5486 && session
5487 .membership_head
5488 .as_ref()
5489 .is_none_or(|(head, _)| *head <= owner.epoch())
5490 });
5491 if committed_here || reached_head {
5492 membership::announce_head(session);
5493 }
5494 value["activity"] = activity_value(session);
5495 value
5496 } else if let Request::RetainedAdmission {
5497 authenticated_endpoint,
5498 request,
5499 } = request
5500 {
5501 let workspace = session
5502 .workspace
5503 .as_ref()
5504 .ok_or("session has no workspace")?;
5505 retained_reply(workspace, authenticated_endpoint, &request)?
5506 } else if matches!(request, Request::IssueInvitation {}) {
5507 let workspace = session
5508 .workspace
5509 .as_ref()
5510 .ok_or("session has no workspace")?;
5511 let (invitation, checkpoint) = workspace.issue_invitation().map_err(str::to_owned)?;
5512 invitation_envelope(session, &invitation, checkpoint)?
5513 } else if let Request::InspectInvitation {
5514 invitation,
5515 checkpoint,
5516 } = request
5517 {
5518 inspected_invitation(&invitation, &checkpoint)?
5519 } else if let Request::BeginJoin {
5520 invitation,
5521 checkpoint,
5522 display_name,
5523 peers,
5524 } = request
5525 {
5526 if session.workspace.is_some() || session.pending_join.is_some() {
5527 return Err("session already owns workspace state".into());
5528 }
5529 let invitation =
5530 arachne_security::Invitation::from_bytes(&invitation).map_err(str::to_owned)?;
5531 let pending = if checkpoint.is_empty() {
5532 arachne_security::PendingJoin::from_compact_invitation(
5533 &invitation,
5534 session.node.id(),
5535 &display_name,
5536 )
5537 } else {
5538 arachne_security::PendingJoin::from_invitation(
5539 &invitation,
5540 &checkpoint,
5541 session.node.id(),
5542 &display_name,
5543 )
5544 }
5545 .map_err(str::to_owned)?;
5546 transition_activity(session, WorkspacePhase::Joining, None)?;
5547 let value = pending_metadata(&pending, session.node.id())?;
5548 let mut value = value;
5549 value["activity"] = activity_value(session);
5550 session.pending_join = Some(pending);
5551 session.join_lifecycle = if peers.is_empty() {
5552 None
5553 } else {
5554 Some(JoinLifecycle::new(peers)?)
5555 };
5556 session.join_history_prefix.clear();
5557 value
5558 } else if matches!(request, Request::SealPendingJoin {}) {
5559 let pending = session
5560 .pending_join
5561 .as_ref()
5562 .ok_or("session has no pending join")?;
5563 let key = session
5564 .storage_key
5565 .as_ref()
5566 .ok_or("session has no protected root key")?;
5567 json!({"workspace": pending.workspace_id(), "snapshot": pending.seal(key).map_err(str::to_owned)?})
5568 } else if let Request::RestorePendingJoin {
5569 workspace,
5570 snapshot,
5571 } = request
5572 {
5573 if session.workspace.is_some() || session.pending_join.is_some() {
5574 return Err("session already owns workspace state".into());
5575 }
5576 let key = session
5577 .storage_key
5578 .as_ref()
5579 .ok_or("session has no protected root key")?;
5580 let pending =
5581 arachne_security::PendingJoin::restore(key, session.node.id(), workspace, &snapshot)
5582 .map_err(str::to_owned)?;
5583 let value = pending_metadata(&pending, session.node.id())?;
5584 transition_activity(session, WorkspacePhase::Joining, None)?;
5585 let mut value = value;
5586 value["activity"] = activity_value(session);
5587 session.pending_join = Some(pending);
5588 session.join_lifecycle = None;
5589 session.join_history_prefix.clear();
5590 value
5591 } else if let Request::CreateWorkspace {
5592 display_name,
5593 workspace_name,
5594 } = request
5595 {
5596 if session.workspace.is_some() || session.pending_join.is_some() {
5597 return Err("session already owns a workspace".into());
5598 }
5599 transition_activity(session, WorkspacePhase::Creating, None)?;
5600 let workspace = arachne_security::Workspace::create_named(
5601 session.node.id(),
5602 &display_name,
5603 workspace_name.as_deref(),
5604 )
5605 .map_err(|error| {
5606 let _ = transition_activity(session, WorkspacePhase::Failed, Some("create_failed"));
5607 error.to_string()
5608 })?;
5609 transition_activity(session, WorkspacePhase::Active, None)?;
5610 let value = json!({"workspace": workspace.id(), "workspace_name":workspace.workspace_name().map_err(str::to_owned)?, "epoch": workspace.epoch(),
5611 "members": workspace.member_count(), "member": member_metadata(&workspace), "durable": false});
5612 let mut value = value;
5613 commit_workspace(session, workspace);
5614 value["activity"] = activity_value(session);
5615 value
5616 } else if matches!(request, Request::SealWorkspace {}) {
5617 if session.records.is_some() {
5618 return Err(
5619 "native records already own persistence; save staged candidates directly".into(),
5620 );
5621 }
5622 if session.staged_workspace.is_some()
5623 || session.inbound_admission.is_some()
5624 || !session.queued_admission_in_flight.is_empty()
5625 {
5626 return Err("admission candidate awaits durable adoption or reply".into());
5627 }
5628 let workspace = session
5629 .workspace
5630 .as_ref()
5631 .ok_or("session has no workspace")?;
5632 let key = session
5633 .storage_key
5634 .as_ref()
5635 .ok_or("session has no protected root key")?;
5636 let snapshot = seal_state(
5637 session.records.is_some(),
5638 workspace,
5639 key,
5640 session.publisher.as_ref(),
5641 session.received.as_ref(),
5642 session.inbox.as_ref(),
5643 )?;
5644 json!({"workspace": workspace.id(), "snapshot": snapshot})
5645 } else if let Request::RestoreWorkspace {
5646 workspace,
5647 snapshot,
5648 } = request
5649 {
5650 if session.workspace.is_some() || session.pending_join.is_some() {
5651 return Err("session already owns a workspace".into());
5652 }
5653 let key = session
5654 .storage_key
5655 .as_ref()
5656 .ok_or("session has no protected root key")?;
5657 if snapshot.starts_with(b"DFRM") {
5658 let removed = arachne_security::RemovedMembership::restore(
5659 key,
5660 session.node.id(),
5661 workspace,
5662 &snapshot,
5663 )
5664 .map_err(str::to_owned)?;
5665 let value = json!({"workspace":removed.workspace_id(), "epoch":removed.epoch(),
5666 "state":"removed", "member":{"id":removed.member().id(), "display_name":removed.member().display_name()},
5667 "commit_digest":removed.commit_digest(), "workspace_ready":false});
5668 *ended = Some(guard.take().ok_or("node is closed")?);
5672 return Ok(value);
5673 }
5674 let (restored, publisher, received, inbox) = if snapshot.starts_with(b"DFWB\x01") {
5675 let (_, attachment) = arachne_security::Workspace::restore_with_attachment(
5676 key,
5677 session.node.id(),
5678 workspace,
5679 &snapshot,
5680 )
5681 .map_err(str::to_owned)?;
5682 if attachment.starts_with(b"DFOI\x01") {
5683 let (owner, log, inbox) = arachne_delivery::inbox::ObjectInbox::restore(
5684 key,
5685 session.node.id(),
5686 workspace,
5687 &snapshot,
5688 )
5689 .map_err(str::to_owned)?;
5690 let received = inbox.legacy_receipts().map_err(str::to_owned)?;
5691 (owner, Some(log), received, Some(inbox))
5692 } else {
5693 let (owner, log, received) = arachne_delivery::PublisherLog::restore_with_receipts(
5694 key,
5695 session.node.id(),
5696 workspace,
5697 &snapshot,
5698 )
5699 .map_err(str::to_owned)?;
5700 let (_, attachment) = arachne_security::Workspace::restore_with_attachment(
5701 key,
5702 session.node.id(),
5703 workspace,
5704 &snapshot,
5705 )
5706 .map_err(str::to_owned)?;
5707 let received = attachment.starts_with(b"DFDL\x01").then_some(received);
5708 (owner, Some(log), received, None)
5709 }
5710 } else {
5711 (
5712 arachne_security::Workspace::restore(key, session.node.id(), workspace, &snapshot)
5713 .map_err(str::to_owned)?,
5714 None,
5715 None,
5716 None,
5717 )
5718 };
5719 transition_activity(session, WorkspacePhase::Active, None)?;
5720 let value = json!({"workspace": restored.id(), "workspace_name":restored.workspace_name().map_err(str::to_owned)?, "epoch": restored.epoch(),
5721 "workspace_name_missing_history":restored.workspace_name_missing_history().map_err(str::to_owned)?,
5722 "members": restored.member_count(), "member": member_metadata(&restored), "durable": false});
5723 session.publisher = publisher;
5724 session.received = received;
5725 session.inbox = inbox;
5726 commit_workspace(session, restored);
5727 let mut value = value;
5728 value["activity"] = activity_value(session);
5729 value
5730 } else if matches!(
5731 request,
5732 Request::Subscribe { .. } | Request::Unsubscribe { .. }
5733 ) && !session.interests.is_idle()
5734 {
5735 return Err("interest repair is active; use set_interest".into());
5737 } else if let Request::SetInterest {
5738 workspace,
5739 revision,
5740 topic,
5741 subscribed,
5742 } = request
5743 {
5744 session.interests.set(
5745 &session.node,
5746 &session.runtime,
5747 interest::Update {
5748 workspace,
5749 revision,
5750 topic,
5751 subscribed,
5752 },
5753 )?
5754 } else if matches!(request, Request::PollInterest {}) {
5755 session.interests.poll(&session.node, &session.runtime)
5756 } else if matches!(request, Request::Poll {}) {
5757 match session.receiver.try_recv() {
5758 Ok(message) => json!({"workspace": message.workspace, "revision": message.revision,
5759 "sender": message.sender, "topic": message.topic.as_str(), "payload": message.payload}),
5760 Err(mpsc::error::TryRecvError::Empty) => Value::Null,
5761 Err(mpsc::error::TryRecvError::Disconnected) => {
5762 return Err("event receiver closed".into());
5763 }
5764 }
5765 } else {
5766 session.runtime.block_on(async {
5767 tokio::time::timeout(Duration::from_secs(10), async {
5768 let node = &session.node;
5769 let result = match request {
5770 Request::AddAddressHint { peer, address } => {
5771 node.add_address_hint(
5772 peer,
5773 address.parse().map_err(|_| "invalid socket address")?,
5774 )
5775 .await
5776 .map_err(|e| e.to_string())?;
5777 session.interests.repair();
5778 Value::Null
5779 }
5780 Request::InstallWorkspacePolicy { revision } => {
5781 let owner = session.workspace.as_ref().ok_or("session has no workspace")?;
5782 if owner.epoch().checked_add(1) != Some(revision) {
5783 return Err("workspace policy revision must match current epoch".into());
5784 }
5785 let policy = owner.member_endpoints().map_err(str::to_owned)?.into_iter()
5786 .map(|endpoint| (endpoint, Permissions::AllTopics)).collect();
5787 install_gossip_policy(
5788 node,
5789 &mut session.overlay_paths,
5790 owner.id(),
5791 revision,
5792 policy,
5793 )
5794 .await?;
5795 session.interests.replace_revision(owner.id(), revision);
5796 json!({"workspace":owner.id(), "revision":revision, "members":owner.member_count()})
5797 }
5798 Request::InstallMemberPolicy { revision, topics } => {
5799 let workspace = session
5800 .workspace
5801 .as_ref()
5802 .ok_or("session has no workspace")?;
5803 let topics = topics
5804 .into_iter()
5805 .map(Topic::new)
5806 .collect::<Result<BTreeSet<_>, _>>()
5807 .map_err(|e| e.to_string())?;
5808 if topics.is_empty() {
5809 return Err("member policy requires an explicit topic set".into());
5810 }
5811 let mut policy = BTreeMap::new();
5812 for endpoint in workspace.member_endpoints().map_err(str::to_owned)? {
5813 if policy
5814 .insert(
5815 endpoint,
5816 Permissions::Selected {
5817 publish: topics.clone(),
5818 subscribe: topics.clone(),
5819 },
5820 )
5821 .is_some()
5822 {
5823 return Err("duplicate member endpoint".into());
5824 }
5825 }
5826 install_gossip_policy(
5827 node,
5828 &mut session.overlay_paths,
5829 workspace.id(),
5830 revision,
5831 policy,
5832 )
5833 .await?;
5834 session.interests.replace_revision(workspace.id(), revision);
5835 json!({"workspace":workspace.id(), "revision":revision,
5836 "members":workspace.member_count()})
5837 }
5838 Request::InstallVerifiedPolicy {
5839 workspace,
5840 revision,
5841 endpoints,
5842 } => {
5843 let mut policy = BTreeMap::new();
5844 for endpoint in endpoints {
5845 let access = Permissions::Selected {
5846 publish: endpoint
5847 .publish
5848 .into_iter()
5849 .map(Topic::new)
5850 .collect::<Result<BTreeSet<_>, _>>()
5851 .map_err(|e| e.to_string())?,
5852 subscribe: endpoint
5853 .subscribe
5854 .into_iter()
5855 .map(Topic::new)
5856 .collect::<Result<BTreeSet<_>, _>>()
5857 .map_err(|e| e.to_string())?,
5858 };
5859 if policy.insert(endpoint.peer, access).is_some() {
5860 return Err("duplicate endpoint".into());
5861 }
5862 }
5863 node.install_verified_policy(workspace, revision, policy)
5864 .await
5865 .map_err(|e| e.to_string())?;
5866 session.interests.replace_revision(workspace, revision);
5867 Value::Null
5868 }
5869 Request::Subscribe {
5870 workspace,
5871 revision,
5872 topic,
5873 } => report(
5874 node.subscribe(
5875 workspace,
5876 revision,
5877 Topic::new(topic).map_err(|e| e.to_string())?,
5878 )
5879 .await
5880 .map_err(|e| e.to_string())?,
5881 ),
5882 Request::Unsubscribe {
5883 workspace,
5884 revision,
5885 topic,
5886 } => report(
5887 node.unsubscribe(
5888 workspace,
5889 revision,
5890 Topic::new(topic).map_err(|e| e.to_string())?,
5891 )
5892 .await
5893 .map_err(|e| e.to_string())?,
5894 ),
5895 Request::Publish {
5896 workspace,
5897 revision,
5898 topic,
5899 payload,
5900 } => report(
5901 node.publish(
5902 workspace,
5903 revision,
5904 Topic::new(topic).map_err(|e| e.to_string())?,
5905 payload,
5906 )
5907 .await
5908 .map_err(|e| e.to_string())?,
5909 ),
5910 Request::Resource { .. }
5911 | Request::WorkspaceMetrics {}
5912 | Request::WorkspaceState {}
5913 | Request::ResetWorkspace {}
5914 | Request::NetworkChange {}
5915 | Request::NearbyEndpoints {}
5916 | Request::SetNearbyIdentity { .. }
5917 | Request::NearbyWorkspaces {}
5918 | Request::SetNearbyWorkspace { .. }
5919 | Request::SendNearbyInvitation { .. }
5920 | Request::EndpointInfo {}
5921 | Request::ControlExchange { .. }
5922 | Request::StageNetworkPublication { .. }
5923 | Request::PollProtected {}
5924 | Request::StagePublication { .. }
5925 | Request::AdoptPublication { .. }
5926 | Request::StageReception { .. }
5927 | Request::AdoptReception { .. }
5928 | Request::AdoptRecovery { .. }
5929 | Request::JoinViaPeer { .. }
5930 | Request::DriveJoin {}
5931 | Request::FetchInvitationCheckpoint { .. }
5932 | Request::DiscoverRecoveryCutoff { .. }
5933 | Request::PollRecoveryCutoff {}
5934 | Request::FetchRecoveryRange { .. }
5935 | Request::PollRecoveryRange {}
5936 | Request::StageRecoveryRange { .. }
5937 | Request::NextDirectGap {}
5938 | Request::FetchDirectRecovery { .. }
5939 | Request::PollDirectRecovery {}
5940 | Request::StageDirectRecovery {}
5941 | Request::StageDirectMiss {}
5942 | Request::CancelDirectRecovery {}
5943 | Request::FetchCurrentView { .. }
5944 | Request::PollCurrentView {}
5945 | Request::StageCurrentView {}
5946 | Request::AdoptCurrentView { .. }
5947 | Request::CancelCurrentView {}
5948 | Request::PollRecoveredPublication {}
5949 | Request::CancelRecoveryRange {}
5950 | Request::PollAdmission { .. }
5951 | Request::DriveWorkspace {}
5952 | Request::ListAdmissionApprovals { .. }
5953 | Request::AcknowledgeAdmissionApproval { .. }
5954 | Request::SendAdmissionReply {}
5955 | Request::StageJoin { .. }
5956 | Request::AdoptJoin { .. }
5957 | Request::FetchMembershipUpdate { .. }
5958 | Request::PollMembershipUpdate {}
5959 | Request::PollWorkspacePresence { .. }
5960 | Request::NextMembershipPeer { .. }
5961 | Request::OfferMembershipUpdate { .. }
5962 | Request::OfferStagedMembershipUpdate { .. }
5963 | Request::PollMembershipOffer {}
5964 | Request::DiscardWorkspaceCandidate {}
5965 | Request::MemberRoster { .. }
5966 | Request::UseServiceProfile {}
5967 | Request::LeaveViaPeer { .. }
5968 | Request::StageSoloLeave {}
5969 | Request::StageInvitation { .. }
5970 | Request::StageInvitationApproval { .. }
5971 | Request::StageInvitationDecline { .. }
5972 | Request::InvitationControls {}
5973 | Request::StageManagement { .. }
5974 | Request::StageWorkspaceName { .. }
5975 | Request::StageWorkspaceNameUpdate { .. }
5976 | Request::StageWorkspaceNameCheckpoint { .. }
5977 | Request::InspectInvitation { .. }
5978 | Request::StageAdmissionUpdate { .. }
5979 | Request::StageAdmission { .. }
5980 | Request::AdoptAdmission { .. }
5981 | Request::RetainedAdmission { .. }
5982 | Request::Poll {}
5983 | Request::IssueInvitation {}
5984 | Request::BeginJoin { .. }
5985 | Request::SealPendingJoin {}
5986 | Request::RestorePendingJoin { .. }
5987 | Request::CreateWorkspace { .. }
5988 | Request::SealWorkspace {}
5989 | Request::EnableObjectDelivery {}
5990 | Request::PollPendingObject { .. }
5991 | Request::StageObjectAcknowledgement { .. }
5992 | Request::StageObjectRejection { .. }
5993 | Request::RestoreWorkspace { .. }
5994 | Request::SetInterest { .. }
5995 | Request::PollInterest {} => unreachable!(),
5996 };
5997 Ok::<_, String>(result)
5998 })
5999 .await
6000 .map_err(|_| "operation deadline exceeded; outcome may be partial".to_string())?
6001 })?
6002 };
6003 Ok(value)
6004}
6005
6006#[cfg(test)]
6007mod tests {
6008 use super::*;
6009
6010 #[test]
6011 fn join_lifecycle_retries_after_the_last_peer() {
6012 let peers = vec![[1; 32], [2; 32]];
6013 let mut lifecycle = JoinLifecycle::new(peers.clone()).unwrap();
6014 lifecycle.selected = Some(peers[0]);
6015 lifecycle.advance();
6016 assert_eq!(lifecycle.cursor, 1);
6017 assert_eq!(lifecycle.selected, None);
6018 lifecycle.selected = Some(peers[1]);
6019 lifecycle.advance();
6020 assert_eq!(lifecycle.cursor, 0);
6021 assert_eq!(lifecycle.selected, None);
6022 }
6023
6024 #[test]
6025 fn admission_batch_wire_sizes_fit_transport_bounds() {
6026 use arachne_security::{
6027 AdmissionAssessment, MembershipAuthorization, PendingJoin, Workspace,
6028 };
6029
6030 for count in [8, MAX_RUNTIME_ADMISSION_BATCH] {
6031 let endpoint = |index: usize| {
6032 let mut value = [0; 32];
6033 value[..8].copy_from_slice(&(index as u64 + 1).to_be_bytes());
6034 value
6035 };
6036 let mut owner = Workspace::create(endpoint(10_000), "Wire-size owner").unwrap();
6037 let (invitation, checkpoint) = owner.issue_invitation().unwrap();
6038 let mut joins = Vec::with_capacity(count);
6039 let mut requests = Vec::with_capacity(count);
6040 let mut validated = Vec::with_capacity(count);
6041 for index in 0..count {
6042 let join = PendingJoin::from_invitation(
6043 &invitation,
6044 &checkpoint,
6045 endpoint(index + 20_000),
6046 "Wire-size member",
6047 )
6048 .unwrap();
6049 let request = join.admission_request().unwrap().to_vec();
6050 let validated_request = match owner
6051 .assess_admission(endpoint(index + 20_000), &request)
6052 .unwrap()
6053 {
6054 AdmissionAssessment::Ready(request) => request,
6055 _ => panic!("wire-size invitation unexpectedly needs approval"),
6056 };
6057 joins.push(join);
6058 requests.push(request);
6059 validated.push(validated_request);
6060 }
6061 let entries: Vec<_> = (0..count)
6062 .map(|index| {
6063 (
6064 endpoint(index + 20_000),
6065 requests[index].as_slice(),
6066 &validated[index],
6067 )
6068 })
6069 .collect();
6070 let prepared = owner.prepare_validated_admission_batch(&entries).unwrap();
6071 let authorization = MembershipAuthorization::AdmissionBatch(
6072 prepared
6073 .replies
6074 .iter()
6075 .map(|reply| reply.authorization.clone())
6076 .collect(),
6077 );
6078 let step = membership::step_json(&authorization, &prepared.commit);
6079 let mut offer = b"DFMO\x01".to_vec();
6080 offer.extend(prepared.workspace.id());
6081 offer.extend(0_u64.to_be_bytes());
6082 offer.extend(serde_json::to_vec(&step).unwrap());
6083 let mut reply =
6084 retained_reply(&prepared.workspace, endpoint(20_000), &requests[0]).unwrap();
6085 reply["commits"] = json!([step]);
6086 let reply = serde_json::to_vec(&reply).unwrap();
6087 println!(
6088 "admission_wire_size count={count} offer_bytes={} reply_bytes={}",
6089 offer.len(),
6090 reply.len()
6091 );
6092 assert!(
6093 offer.len() <= 32 * 1024,
6094 "offer exceeds control request bound"
6095 );
6096 assert!(
6097 reply.len() <= arachne_node::MAX_CONTROL_REPLY,
6098 "reply exceeds control response bound"
6099 );
6100 owner = prepared.workspace;
6101 assert_eq!(owner.member_count(), count + 1);
6102 assert_eq!(joins.len(), count);
6103 }
6104 }
6105
6106 #[test]
6107 #[ignore = "explicit 500-member runtime history-page capacity run"]
6108 fn admission_history_pages_handle_500_member_burst() {
6109 use arachne_security::{AdmissionAssessment, PendingJoin, Workspace};
6110
6111 let started = std::time::Instant::now();
6112 let endpoint = |index: usize| {
6113 let mut value = [0; 32];
6114 value[..8].copy_from_slice(&(index as u64 + 1).to_be_bytes());
6115 value
6116 };
6117 let mut owner = Workspace::create(endpoint(10_000), "500-member owner").unwrap();
6118 let (invitation, checkpoint) = owner.issue_invitation().unwrap();
6119 let mut joins = Vec::with_capacity(500);
6120 let mut requests = Vec::with_capacity(500);
6121 let mut validated = Vec::with_capacity(500);
6122 for index in 0..500 {
6123 let remote = endpoint(index + 20_000);
6124 let join =
6125 PendingJoin::from_invitation(&invitation, &checkpoint, remote, "Burst member")
6126 .unwrap();
6127 let request = join.admission_request().unwrap().to_vec();
6128 let validated_request = match owner.assess_admission(remote, &request).unwrap() {
6129 AdmissionAssessment::Ready(request) => request,
6130 _ => panic!("500-member invitation unexpectedly needs approval"),
6131 };
6132 joins.push(join);
6133 requests.push(request);
6134 validated.push(validated_request);
6135 }
6136
6137 let mut welcome = Vec::new();
6138 let mut batches = 0;
6139 for start in (0..500).step_by(MAX_RUNTIME_ADMISSION_BATCH) {
6140 let end = (start + MAX_RUNTIME_ADMISSION_BATCH).min(500);
6141 let entries: Vec<_> = (start..end)
6142 .map(|index| {
6143 (
6144 endpoint(index + 20_000),
6145 requests[index].as_slice(),
6146 &validated[index],
6147 )
6148 })
6149 .collect();
6150 let prepared = owner.prepare_validated_admission_batch(&entries).unwrap();
6151 welcome = prepared.welcome.clone();
6152 owner = prepared.workspace;
6153 batches += 1;
6154 if batches % 8 == 0 || end == 500 {
6155 eprintln!("memory members={end} {:?}", owner.memory_report());
6156 }
6157 }
6158
6159 let target = joins.last().unwrap();
6160 let target_endpoint = endpoint(20_499);
6161 let target_request = target.admission_request().unwrap();
6162 let mut offset = 0;
6163 let mut pages = 0;
6164 let mut commits = Vec::new();
6165 let mut page_ms = Vec::new();
6166 let mut page_bytes = Vec::new();
6167 loop {
6168 let page_started = std::time::Instant::now();
6169 let encoded = admission_reply_page(
6170 &owner,
6171 target_endpoint,
6172 target_request,
6173 Some(&checkpoint),
6174 offset,
6175 )
6176 .unwrap();
6177 page_ms.push(page_started.elapsed().as_secs_f64() * 1000.0);
6178 page_bytes.push(encoded.len());
6179 assert!(encoded.len() <= arachne_node::MAX_CONTROL_REPLY);
6180 let page: Value = serde_json::from_slice(&encoded).unwrap();
6181 commits.extend(page["commits"].as_array().unwrap().iter().cloned());
6182 pages += 1;
6183 if page["history_complete"].as_bool().unwrap() {
6184 break;
6185 }
6186 offset = page["history_next"].as_u64().unwrap() as usize;
6187 }
6188 assert_eq!(commits.len(), batches);
6189
6190 let mut proof = target.join_proof().unwrap();
6191 for commit in commits {
6192 let step: JoinStep = serde_json::from_value(commit).unwrap();
6193 proof
6194 .apply_transition(&step.authorization().unwrap(), &step.commit)
6195 .unwrap();
6196 }
6197 let joined = target.prepare_workspace(&proof, &welcome).unwrap();
6198 assert_eq!(joined.member_count(), 501);
6199 println!(
6200 "admission_runtime_capacity members=500 batches={batches} pages={pages} elapsed_ms={} page_ms={page_ms:.1?} page_bytes={page_bytes:?}",
6201 started.elapsed().as_millis()
6202 );
6203 }
6204
6205 #[test]
6206 fn pending_object_query_bounds_and_gap_epoch_guard() {
6207 use arachne_delivery::{
6208 PublisherLog,
6209 inbox::{InboxStage, ObjectInbox},
6210 };
6211 use arachne_routing::PublicationContext;
6212 use arachne_security::{PendingJoin, StorageKey, Workspace};
6213
6214 let root = [103; 32];
6215 let handle = create(Some(&root)).unwrap();
6216 let call = |request: Value| -> Result<Value, String> {
6217 serde_json::from_slice(&execute(handle, &serde_json::to_vec(&request).unwrap())?)
6218 .map_err(|error| error.to_string())
6219 };
6220 let description: Value = serde_json::from_str(&describe(handle).unwrap()).unwrap();
6221 let endpoint = serde_json::from_value(description["endpoint_key"].clone()).unwrap();
6222 let admin = Workspace::create([104; 32], "Publisher").unwrap();
6223 let (invitation, checkpoint) = admin.issue_invitation().unwrap();
6224 let join =
6225 PendingJoin::from_invitation(&invitation, &checkpoint, endpoint, "Reader").unwrap();
6226 let prepared = admin
6227 .prepare_admission(endpoint, join.admission_request().unwrap())
6228 .unwrap();
6229 let mut proof = join.join_proof().unwrap();
6230 proof
6231 .apply_add(&prepared.authorization, &prepared.commit)
6232 .unwrap();
6233 let reader = join.prepare_workspace(&proof, &prepared.welcome).unwrap();
6234 let mut sender = prepared.workspace;
6235 let mut inbox = ObjectInbox::new(reader.id(), reader.epoch());
6236 for direct in [true, false] {
6237 let context = PublicationContext {
6238 workspace: reader.id(),
6239 revision: 7,
6240 topic: Topic::new("chat/messages").unwrap(),
6241 id: [if direct { 1 } else { 2 }; 16],
6242 sequence: std::num::NonZeroU64::new(2),
6243 };
6244 let recipients = if direct {
6245 vec![reader.member().unwrap().id()]
6246 } else {
6247 vec![]
6248 };
6249 let aad = if direct {
6250 context.direct_authenticated_bytes(&recipients).unwrap()
6251 } else {
6252 context.authenticated_bytes()
6253 };
6254 let object = sender.protect_object(&aad, b"pending").unwrap();
6255 let InboxStage::Prepared(next) = inbox
6256 .stage_with_recipients(&reader, &context, &recipients, &object)
6257 .unwrap()
6258 else {
6259 panic!("object was not staged")
6260 };
6261 inbox = *next;
6262 }
6263 let publisher =
6264 PublisherLog::new(reader.id(), reader.member().unwrap().id(), reader.epoch());
6265 let snapshot = inbox
6266 .seal(&reader, &StorageKey::derive(&root).unwrap(), &publisher)
6267 .unwrap();
6268 call(json!({"op":"restore_workspace","workspace":reader.id(),"snapshot":snapshot}))
6269 .unwrap();
6270 let pending = call(json!({"op":"poll_pending_object"})).unwrap();
6271 assert_eq!(pending["id"], json!(vec![2; 16]));
6272 assert_eq!(
6273 call(json!({"op":"poll_pending_object","deferred":[]})).unwrap(),
6274 pending
6275 );
6276 let scope = json!({"member":pending["member"], "revision":pending["revision"],
6277 "topic":pending["topic"], "recipients":pending["recipients"]});
6278 let poll = |deferred: Value| call(json!({"op":"poll_pending_object","deferred":deferred}));
6279 assert_eq!(poll(json!([scope])).unwrap(), Value::Null);
6281 assert_eq!(poll(json!(vec![scope.clone(); 64])).unwrap(), Value::Null);
6282 assert_eq!(
6283 poll(json!(vec![scope.clone(); 65])).unwrap_err(),
6284 "too many deferred delivery streams"
6285 );
6286 let mut audience = scope.clone();
6287 audience["recipients"] = json!((0..64u8).map(|number| [number; 32]).collect::<Vec<_>>());
6288 assert_eq!(poll(json!([audience])).unwrap(), pending);
6289 for (field, value) in [
6290 ("revision", json!(0)),
6291 ("topic", json!("")),
6292 ("topic", json!("chat//messages")),
6293 ("topic", json!("a".repeat(129))),
6294 ("recipients", json!(vec![[1; 32], [1; 32]])),
6295 ("recipients", json!(vec![[2; 32], [1; 32]])),
6296 (
6297 "recipients",
6298 json!((0..65u8).map(|number| [number; 32]).collect::<Vec<_>>()),
6299 ),
6300 ] {
6301 let mut invalid = scope.clone();
6302 invalid[field] = value;
6303 assert_eq!(
6304 poll(json!([invalid])).unwrap_err(),
6305 "invalid deferred delivery stream"
6306 );
6307 }
6308 for (field, value) in [
6309 ("member", json!(vec![1; 31])),
6310 ("author", json!(vec![1; 32])),
6311 ("revision", json!(-1)),
6312 ("recipients", json!(vec![[1; 31]])),
6313 ] {
6314 let mut invalid = scope.clone();
6315 invalid[field] = value;
6316 assert!(poll(json!([invalid])).is_err());
6317 }
6318 assert!(poll(Value::Null).is_err());
6319 assert_eq!(call(json!({"op":"poll_pending_object"})).unwrap(), pending);
6320 {
6321 let shared = session(handle).unwrap();
6322 let mut locked = shared.lock().unwrap();
6323 assert!(check_epoch_transition(locked.as_mut().unwrap()).is_err());
6324 }
6325 let staged = call(
6326 json!({"op":"stage_object_acknowledgement", "member":pending["member"],
6327 "topic":pending["topic"], "counter":pending["counter"], "id":pending["id"]}),
6328 )
6329 .unwrap();
6330 call(json!({"op":"adopt_reception","snapshot":staged["snapshot"]})).unwrap();
6331 assert_eq!(
6332 call(json!({"op":"poll_pending_object"})).unwrap(),
6333 Value::Null
6334 );
6335 {
6336 let shared = session(handle).unwrap();
6337 let mut locked = shared.lock().unwrap();
6338 let owner = locked.as_mut().unwrap();
6339 assert_eq!(owner.inbox.as_ref().unwrap().pending_count(), 1);
6340 assert_eq!(
6341 check_epoch_transition(owner).unwrap_err(),
6342 "pending application delivery must be acknowledged before membership update"
6343 );
6344 }
6345 close(handle).unwrap();
6346 }
6347
6348 #[test]
6349 fn overlay_path_budget_fails_closed_and_releases() {
6350 let total = AtomicUsize::new(0);
6351 assert!(reserve_overlay_paths(&total, MAX_DEVICE_OVERLAY_PATHS));
6352 assert!(!reserve_overlay_paths(&total, 1));
6353 release_overlay_paths(&total, MAX_DEVICE_OVERLAY_PATHS);
6354 assert_eq!(total.load(Ordering::Acquire), 0);
6355 assert!(reserve_overlay_paths(&total, 5));
6356 assert_eq!(total.load(Ordering::Acquire), 5);
6357 }
6358
6359 #[test]
6360 fn real_node_lifecycle_rejects_stale_handles_and_releases_capacity() {
6361 assert!(describe(0).is_err());
6362 assert!(close(-1).is_err());
6363 let handles: Vec<_> = (0..8).map(|_| create(None).unwrap()).collect();
6364 assert!(create(None).is_err());
6365 for handle in &handles {
6366 let info: serde_json::Value =
6367 serde_json::from_str(&describe(*handle).unwrap()).unwrap();
6368 assert_eq!(info["endpoint_key"].as_array().unwrap().len(), 32);
6369 assert_eq!(info["workspace_ready"], false);
6370 close(*handle).unwrap();
6371 assert!(describe(*handle).is_err());
6372 assert!(close(*handle).is_err());
6373 }
6374 let next = create(None).unwrap();
6375 assert!(next > *handles.last().unwrap());
6376 close(next).unwrap();
6377
6378 let a = create(None).unwrap();
6380 let b = create(None).unwrap();
6381 let ai: Value = serde_json::from_str(&describe(a).unwrap()).unwrap();
6382 let bi: Value = serde_json::from_str(&describe(b).unwrap()).unwrap();
6383 let call = |handle, value: Value| -> Result<Value, String> {
6384 serde_json::from_slice(&execute(handle, &serde_json::to_vec(&value).unwrap())?)
6385 .map_err(|e| e.to_string())
6386 };
6387 assert_eq!(
6388 call(a, json!({"op":"network_change"})).unwrap(),
6389 json!({"notified":true})
6390 );
6391 for (handle, other) in [(a, &bi), (b, &ai)] {
6392 let port = other["bound_address"]
6393 .as_str()
6394 .unwrap()
6395 .parse::<std::net::SocketAddr>()
6396 .unwrap()
6397 .port();
6398 call(
6399 handle,
6400 json!({"op":"add_address_hint", "peer":other["endpoint_key"],
6401 "address":format!("127.0.0.1:{port}")}),
6402 )
6403 .unwrap();
6404 call(
6405 handle,
6406 json!({"op":"install_verified_policy", "workspace":vec![7;32], "revision":1,
6407 "endpoints":[
6408 {"peer":ai["endpoint_key"], "publish":["streams/sample"], "subscribe":[]},
6409 {"peer":bi["endpoint_key"], "publish":[], "subscribe":["streams/sample"]}
6410 ]}),
6411 )
6412 .unwrap();
6413 }
6414 call(b, json!({"op":"subscribe", "workspace":vec![7;32], "revision":1, "topic":"streams/sample"})).unwrap();
6415 for payload in [vec![0, 255, 17], br#"{"event":"changed"}"#.to_vec()] {
6416 let outcome = call(
6417 a,
6418 json!({"op":"publish", "workspace":vec![7;32], "revision":1,
6419 "topic":"streams/sample", "payload":payload}),
6420 )
6421 .unwrap();
6422 assert_eq!(outcome["admitted"], json!([bi["endpoint_key"]]));
6423 assert_eq!(outcome["failed"], json!([]));
6424 let received = call(b, json!({"op":"poll"})).unwrap();
6425 assert_eq!(received["payload"], json!(payload));
6426 assert_eq!(received["workspace"], json!(vec![7; 32]));
6427 assert_eq!(received["sender"], ai["endpoint_key"]);
6428 }
6429 assert_eq!(call(b, json!({"op":"poll"})).unwrap(), Value::Null);
6430 assert!(
6431 call(
6432 b,
6433 json!({"op":"publish", "workspace":vec![7;32], "revision":1,
6434 "topic":"streams/sample", "payload":[1]})
6435 )
6436 .is_err()
6437 );
6438 assert!(call(a, json!({"op":"poll", "unexpected":true})).is_err());
6439 assert!(execute(a, &vec![b' '; MAX_REQUEST + 1]).is_err());
6440 call(b, json!({"op":"unsubscribe", "workspace":vec![7;32], "revision":1, "topic":"streams/sample"})).unwrap();
6441 assert_eq!(
6442 call(
6443 a,
6444 json!({"op":"publish", "workspace":vec![7;32], "revision":1,
6445 "topic":"streams/sample", "payload":[1]})
6446 )
6447 .unwrap()["admitted"],
6448 json!([])
6449 );
6450 close(a).unwrap();
6451 close(b).unwrap();
6452 assert!(call(a, json!({"op":"poll"})).is_err());
6453
6454 let mut admin = create(Some(&[41; 32])).unwrap();
6455 let joiner = create(Some(&[42; 32])).unwrap();
6456 let created = call(
6457 admin,
6458 json!({"op":"create_workspace","display_name":"Coordinator"}),
6459 )
6460 .unwrap();
6461 let old = call(admin, json!({"op":"seal_workspace"})).unwrap();
6462 let invite = call(admin, json!({"op":"issue_invitation"})).unwrap();
6463 let pending = call(
6464 joiner,
6465 json!({"op":"begin_join","invitation":invite["invitation"],
6466 "checkpoint":invite["checkpoint"],"display_name":"Field member"}),
6467 )
6468 .unwrap();
6469 let stage = json!({"op":"stage_admission","authenticated_endpoint":pending["endpoint"],"request":pending["admission_request"]});
6470 let retry = json!({"op":"retained_admission","authenticated_endpoint":pending["endpoint"],"request":pending["admission_request"]});
6471 let prepared = call(admin, stage.clone()).unwrap();
6472 assert!(prepared.get("welcome").is_none());
6473 assert!(call(admin, retry.clone()).is_err());
6474 assert!(call(admin, json!({"op":"seal_workspace"})).is_err());
6475 assert!(
6476 call(
6477 admin,
6478 json!({"op":"adopt_admission","snapshot":old["snapshot"]})
6479 )
6480 .is_err()
6481 );
6482 close(admin).unwrap(); admin = create(Some(&[41; 32])).unwrap();
6484 let restored = call(admin, json!({"op":"restore_workspace","workspace":created["workspace"],"snapshot":old["snapshot"]})).unwrap();
6485 assert_eq!(restored["members"], 1);
6486 assert!(call(admin, retry.clone()).is_err());
6487 let saved = call(admin, stage).unwrap();
6488 close(admin).unwrap(); admin = create(Some(&[41; 32])).unwrap();
6490 let restored = call(admin, json!({"op":"restore_workspace","workspace":created["workspace"],"snapshot":saved["snapshot"]})).unwrap();
6491 assert_eq!(restored["members"], 2);
6492 assert_eq!(restored["epoch"], 1);
6493 let reply = call(admin, retry.clone()).unwrap();
6494 assert_eq!(reply["welcome"], call(admin, retry).unwrap()["welcome"]);
6495 call(
6496 joiner,
6497 json!({"op":"add_address_hint", "peer":invite["peer"],
6498 "address":serde_json::from_str::<Value>(&describe(admin).unwrap()).unwrap()["bound_address"].as_str().unwrap().replace("0.0.0.0:","127.0.0.1:")}),
6499 )
6500 .unwrap();
6501 let peer = invite["peer"].clone();
6502 let requesting = std::thread::spawn(move || {
6503 let request =
6504 serde_json::to_vec(&json!({"op":"request_admission","peer":peer})).unwrap();
6505 serde_json::from_slice::<Value>(&execute(joiner, &request).unwrap()).unwrap()
6506 });
6507 let mut network_reply = requesting.join().unwrap();
6510 assert_eq!(
6511 call(admin, json!({"op":"poll_admission"})).unwrap(),
6512 Value::Null
6513 );
6514 let pages = network_reply
6517 .as_object_mut()
6518 .unwrap()
6519 .remove("history_page_bytes")
6520 .unwrap();
6521 assert_eq!(pages.as_array().unwrap().len(), 1);
6522 assert!(
6523 pages[0].as_u64().unwrap() as usize <= arachne_node::MAX_CONTROL_REPLY,
6524 "a served page must stay inside the control-reply bound"
6525 );
6526 assert_eq!(
6527 network_reply
6528 .as_object_mut()
6529 .unwrap()
6530 .remove("commits")
6531 .unwrap(),
6532 json!([{"commit":reply["commit"],"authorization":reply["authorization"]}])
6533 );
6534 assert_eq!(network_reply, reply);
6535 let join = json!({"op":"stage_join", "welcome":reply["welcome"],
6536 "commits":[{"commit":reply["commit"], "authorization":reply["authorization"]}]});
6537 let mut invalid = join.clone();
6538 invalid["commits"][0]["authorization"]["grant_signature"] = json!([1]);
6539 assert!(call(joiner, invalid).is_err());
6540 assert!(call(joiner, json!({"op":"seal_pending_join"})).is_ok());
6541 let staged_join = call(joiner, join).unwrap();
6542 assert!(
6543 call(
6544 joiner,
6545 json!({"op":"adopt_admission","snapshot":staged_join["snapshot"]})
6546 )
6547 .is_err()
6548 );
6549 let adopted = call(
6550 joiner,
6551 json!({"op":"adopt_join","snapshot":staged_join["snapshot"]}),
6552 )
6553 .unwrap();
6554 assert_eq!(adopted["members"], 2);
6555 assert!(call(joiner, json!({"op":"seal_pending_join"})).is_err());
6556 close(joiner).unwrap();
6557 let joiner = create(Some(&[42; 32])).unwrap();
6558 let restored = call(joiner,json!({"op":"restore_workspace","workspace":created["workspace"],"snapshot":staged_join["snapshot"]})).unwrap();
6559 assert_eq!(restored["member"], pending["member"]);
6560 let member_policy =
6562 json!({"op":"install_member_policy","revision":17,"topics":["streams/sample"]});
6563 for handle in [admin, joiner] {
6564 assert!(call(handle, json!({"op":"install_verified_policy","workspace":created["workspace"],"revision":17,"endpoints":[]})).is_err());
6565 assert!(
6566 call(
6567 handle,
6568 json!({"op":"install_member_policy","revision":17,"topics":[]})
6569 )
6570 .is_err()
6571 );
6572 assert!(
6573 call(
6574 handle,
6575 json!({"op":"install_member_policy","revision":17,"topics":["bad topic"]})
6576 )
6577 .is_err()
6578 );
6579 let installed = call(handle, member_policy.clone()).unwrap();
6580 assert_eq!(installed["members"], 2);
6581 assert_eq!(installed["revision"], 17);
6582 assert!(call(handle, member_policy.clone()).is_err());
6583 }
6584 let admin_info: Value = serde_json::from_str(&describe(admin).unwrap()).unwrap();
6586 call(joiner, json!({"op":"add_address_hint","peer":admin_info["endpoint_key"],
6587 "address":admin_info["bound_address"].as_str().unwrap().replace("0.0.0.0:","127.0.0.1:")})).unwrap();
6588 let subscribed = call(joiner, json!({"op":"subscribe","workspace":created["workspace"],"revision":17,"topic":"streams/sample"})).unwrap();
6589 assert_eq!(subscribed["admitted"].as_array().unwrap().len(), 2);
6590 assert!(subscribed["failed"].as_array().unwrap().is_empty());
6591 assert!(call(joiner, json!({"op":"subscribe","workspace":created["workspace"],"revision":17,"topic":"streams/other"})).is_err());
6592 let staged_binary = execute_stored(admin, &serde_json::to_vec(&json!({"op":"stage_network_publication","revision":17,"topic":"streams/sample","id":vec![10;16],"payload":[0,255,77]})).unwrap(), &[]).unwrap();
6593 let mut staged_network: Value = serde_json::from_slice(&staged_binary[0]).unwrap();
6594 assert!(staged_network.get("snapshot").is_none());
6595 assert!(!staged_binary[1].is_empty());
6596 staged_network["snapshot"] = json!(staged_binary[1]);
6597 assert!(staged_network.get("ciphertext").is_none());
6598 assert_eq!(
6599 call(joiner, json!({"op":"poll_protected"})).unwrap(),
6600 Value::Null
6601 );
6602 assert!(
6603 call(
6604 admin,
6605 json!({"op":"adopt_publication","snapshot":saved["snapshot"]})
6606 )
6607 .is_err()
6608 );
6609 let snapshot: Vec<u8> = serde_json::from_value(staged_network["snapshot"].clone()).unwrap();
6610 assert!(
6611 execute_stored(
6612 admin,
6613 br#"{"op":"adopt_publication","op":"adopt_publication"}"#,
6614 &snapshot
6615 )
6616 .is_err()
6617 );
6618 let mut damaged = snapshot.clone();
6619 *damaged.last_mut().unwrap() ^= 1;
6620 assert!(execute_stored(admin, br#"{"op":"adopt_publication"}"#, &damaged).is_err());
6621 assert_eq!(
6622 call(joiner, json!({"op":"poll_protected"})).unwrap(),
6623 Value::Null
6624 );
6625 assert!(
6626 execute_stored(
6627 admin,
6628 br#"{"op":"adopt_publication","snapshot":[]}"#,
6629 &snapshot
6630 )
6631 .is_err()
6632 );
6633 assert!(execute_stored(admin, br#"{"op":"endpoint_info"}"#, &snapshot).is_err());
6634 assert!(
6635 execute_stored(
6636 admin,
6637 br#"{"op":"adopt_publication"}"#,
6638 &vec![0; arachne_security::MAX_SEALED_BUNDLE + 1]
6639 )
6640 .is_err()
6641 );
6642 let sent = execute_stored(admin, br#"{"op":"adopt_publication"}"#, &snapshot).unwrap();
6643 assert!(sent[1].is_empty());
6644 let sent: Value = serde_json::from_slice(&sent[0]).unwrap();
6645 assert!(sent["admission"]["admitted"].as_array().unwrap().is_empty());
6646 assert_eq!(sent["admission"]["queued"], true);
6647 assert!(sent["admission"]["failed"].as_array().unwrap().is_empty());
6648 assert!(sent.get("ciphertext").is_none());
6649 let (mut tampered_context, ciphertext) = {
6652 let shared = session(admin).unwrap();
6653 let guard = shared.lock().unwrap();
6654 let log = guard.as_ref().unwrap().publisher.as_ref().unwrap();
6655 let retained = log
6656 .select(
6657 0,
6658 1,
6659 &BTreeSet::from([Topic::new("streams/sample").unwrap()]),
6660 )
6661 .unwrap();
6662 let record = retained.records()[0];
6663 assert_eq!(record.context.sequence.unwrap().get(), record.sequence);
6664 (record.context.clone(), record.ciphertext.clone())
6665 };
6666 tampered_context.sequence = std::num::NonZeroU64::new(2);
6667 assert!(
6668 call(
6669 joiner,
6670 json!({"op":"stage_reception",
6671 "context":tampered_context.authenticated_bytes(), "ciphertext":ciphertext})
6672 )
6673 .is_err()
6674 );
6675 tampered_context.sequence = None;
6676 assert!(
6677 call(
6678 joiner,
6679 json!({"op":"stage_reception",
6680 "context":tampered_context.authenticated_bytes(), "ciphertext":ciphertext})
6681 )
6682 .is_err()
6683 );
6684 let network_received = call(joiner, json!({"op":"poll_protected"})).unwrap();
6685 assert!(network_received.get("payload").is_none());
6686 assert_eq!(network_received["state"], "awaiting_reception_save");
6687 let delivered_network = call(
6688 joiner,
6689 json!({"op":"adopt_reception","snapshot":network_received["snapshot"]}),
6690 )
6691 .unwrap();
6692 assert_eq!(delivered_network["payload"], json!([0, 255, 77]));
6693 assert_eq!(delivered_network["sequence"], 1);
6694 assert_eq!(delivered_network["id"], json!(vec![10; 16]));
6695 assert_eq!(delivered_network["member"], created["member"]["id"]);
6696 assert_eq!(delivered_network["topic"], "streams/sample");
6697 assert!(call(joiner, json!({"op":"poll"})).is_err());
6698 call(admin, json!({"op":"subscribe","workspace":created["workspace"],"revision":17,"topic":"streams/sample"})).unwrap();
6699 let reverse = call(joiner, json!({"op":"stage_network_publication","revision":17,"topic":"streams/sample","id":vec![11;16],"payload":[9]})).unwrap();
6700 let sent = call(
6701 joiner,
6702 json!({"op":"adopt_publication","snapshot":reverse["snapshot"]}),
6703 )
6704 .unwrap();
6705 assert_eq!(sent["admission"]["admitted"].as_array().unwrap().len(), 1);
6706 assert_eq!(sent["admission"]["queued"], true);
6707 assert!(sent["admission"]["failed"].as_array().unwrap().is_empty());
6708 assert_eq!(
6709 call(joiner, json!({"op":"poll_protected"})).unwrap(),
6710 Value::Null
6711 );
6712 let deadline = std::time::Instant::now() + Duration::from_secs(4);
6713 let incoming = loop {
6714 let incoming = call(admin, json!({"op":"poll_protected"})).unwrap();
6715 if !incoming.is_null() {
6716 break incoming;
6717 }
6718 assert!(std::time::Instant::now() < deadline);
6719 std::thread::sleep(Duration::from_millis(5));
6720 };
6721 let reverse_delivered = call(
6722 admin,
6723 json!({"op":"adopt_reception","snapshot":incoming["snapshot"]}),
6724 )
6725 .unwrap();
6726 assert_eq!(reverse_delivered["payload"], json!([9]));
6727 assert_eq!(reverse_delivered["member"], pending["member"]["id"]);
6728 let context = b"workspace/topic/publication".to_vec();
6729 let publication = json!({"op":"stage_publication","context":context,"payload":[0,255,42]});
6730 let staged = call(admin, publication.clone()).unwrap();
6731 assert_eq!(staged["state"], "awaiting_publication_save");
6732 assert!(staged.get("ciphertext").is_none() && staged.get("payload").is_none());
6733 assert!(call(admin, publication.clone()).is_err());
6734 assert!(call(admin, json!({"op":"seal_workspace"})).is_err());
6735 assert!(
6736 call(
6737 admin,
6738 json!({"op":"adopt_reception","snapshot":staged["snapshot"]})
6739 )
6740 .is_err()
6741 );
6742 assert!(
6743 call(
6744 admin,
6745 json!({"op":"adopt_publication","snapshot":saved["snapshot"]})
6746 )
6747 .is_err()
6748 );
6749 let released = call(
6750 admin,
6751 json!({"op":"adopt_publication","snapshot":staged["snapshot"]}),
6752 )
6753 .unwrap();
6754 assert!(released.get("ciphertext").is_some());
6755 assert!(
6756 call(
6757 admin,
6758 json!({"op":"adopt_publication","snapshot":staged["snapshot"]})
6759 )
6760 .is_err()
6761 );
6762 assert!(call(admin, json!({"op":"publish","workspace":created["workspace"],"revision":1,"topic":"sample","payload":[1]})).is_err());
6763 let reception =
6764 json!({"op":"stage_reception","context":context,"ciphertext":released["ciphertext"]});
6765 let mut wrong = reception.clone();
6766 wrong["context"] = json!([1]);
6767 assert!(call(joiner, wrong).is_err());
6768 let received = call(joiner, reception.clone()).unwrap();
6770 assert_eq!(received["state"], "awaiting_reception_save");
6771 assert!(received.get("payload").is_none() && received.get("member").is_none());
6772 assert!(call(joiner, json!({"op":"poll"})).is_err());
6773 assert!(
6774 call(
6775 joiner,
6776 json!({"op":"adopt_reception","snapshot":staged_join["snapshot"]})
6777 )
6778 .is_err()
6779 );
6780 let delivered = call(
6781 joiner,
6782 json!({"op":"adopt_reception","snapshot":received["snapshot"]}),
6783 )
6784 .unwrap();
6785 assert_eq!(delivered["payload"], json!([0, 255, 42]));
6786 assert_eq!(delivered["member"], created["member"]["id"]);
6787 close(joiner).unwrap();
6788 let joiner = create(Some(&[42; 32])).unwrap();
6789 call(joiner, json!({"op":"restore_workspace","workspace":created["workspace"],"snapshot":received["snapshot"]})).unwrap();
6790 assert!(call(joiner, reception).is_err());
6791 let request = serde_json::to_vec(&json!({"op":"stage_network_publication","revision":17,"topic":"streams/sample","id":vec![12;16],"payload":[7]})).unwrap();
6794 let candidate = execute_stored(admin, &request, &[]).unwrap();
6795 assert!(candidate[1].starts_with(b"DFWB\x01"));
6796 let (head, retained) = {
6797 let shared = session(admin).unwrap();
6798 let guard = shared.lock().unwrap();
6799 let session = guard.as_ref().unwrap();
6800 assert_eq!(session.publisher.as_ref().unwrap().head(), 1);
6801 let log = session
6802 .staged_workspace
6803 .as_ref()
6804 .unwrap()
6805 .publisher
6806 .as_ref()
6807 .unwrap();
6808 assert_eq!(log.head(), 2);
6809 let range = log
6810 .select(
6811 1,
6812 2,
6813 &BTreeSet::from([Topic::new("streams/sample").unwrap()]),
6814 )
6815 .unwrap();
6816 (log.head(), range.records()[0].ciphertext.clone())
6817 };
6818 close(admin).unwrap();
6819 admin = create(Some(&[41; 32])).unwrap();
6820 let restore =
6821 serde_json::to_vec(&json!({"op":"restore_workspace","workspace":created["workspace"]}))
6822 .unwrap();
6823 let mut corrupt = candidate[1].clone();
6824 *corrupt.last_mut().unwrap() ^= 1;
6825 assert!(execute_stored(admin, &restore, &corrupt).is_err());
6826 execute_stored(admin, &restore, &candidate[1]).unwrap();
6827 {
6828 let shared = session(admin).unwrap();
6829 let guard = shared.lock().unwrap();
6830 let log = guard.as_ref().unwrap().publisher.as_ref().unwrap();
6831 assert_eq!(log.head(), head);
6832 let range = log
6833 .select(
6834 1,
6835 2,
6836 &BTreeSet::from([Topic::new("streams/sample").unwrap()]),
6837 )
6838 .unwrap();
6839 assert_eq!(range.records()[0].ciphertext, retained);
6840 }
6841 let next = execute_stored(admin, &serde_json::to_vec(&json!({"op":"stage_network_publication","revision":17,"topic":"streams/sample","id":vec![13;16],"payload":[7]})).unwrap(), &[]).unwrap();
6842 assert_ne!(candidate[1], next[1]);
6843 {
6844 let shared = session(admin).unwrap();
6845 let guard = shared.lock().unwrap();
6846 let log = guard
6847 .as_ref()
6848 .unwrap()
6849 .staged_workspace
6850 .as_ref()
6851 .unwrap()
6852 .publisher
6853 .as_ref()
6854 .unwrap();
6855 assert_eq!(log.head(), 3);
6856 let range = log
6857 .select(
6858 2,
6859 3,
6860 &BTreeSet::from([Topic::new("streams/sample").unwrap()]),
6861 )
6862 .unwrap();
6863 assert_ne!(range.records()[0].ciphertext, retained);
6864 let range = log
6865 .select(
6866 1,
6867 3,
6868 &BTreeSet::from([Topic::new("streams/sample").unwrap()]),
6869 )
6870 .unwrap();
6871 let packets: Vec<_> = range.records().iter().map(|r| (*r).clone()).collect();
6872 drop(guard);
6873 let shared = session(joiner).unwrap();
6874 let guard = shared.lock().unwrap();
6875 let mut reader = guard
6877 .as_ref()
6878 .unwrap()
6879 .workspace
6880 .as_ref()
6881 .unwrap()
6882 .provisional_copy()
6883 .unwrap();
6884 for packet in packets {
6885 let context = packet.context.authenticated_bytes();
6886 assert_eq!(
6887 reader
6888 .unprotect_application(&context, &packet.ciphertext)
6889 .unwrap()
6890 .payload,
6891 [7]
6892 );
6893 assert!(
6894 reader
6895 .unprotect_application(&context, &packet.ciphertext)
6896 .is_err()
6897 );
6898 }
6899 }
6900 assert!(call(admin, json!({"op":"poll_admission"})).is_ok());
6903 execute_stored(admin, br#"{"op":"adopt_publication"}"#, &next[1]).unwrap();
6904 call(
6905 admin,
6906 json!({"op":"install_member_policy","revision":17,"topics":["streams/sample"]}),
6907 )
6908 .unwrap();
6909 let (peer, address, query) = {
6911 let shared = session(admin).unwrap();
6912 let guard = shared.lock().unwrap();
6913 let owner = guard.as_ref().unwrap();
6914 let workspace = owner.workspace.as_ref().unwrap();
6915 (
6916 owner.node.id(),
6917 std::net::SocketAddr::from(([127, 0, 0, 1], owner.node.address().port())),
6918 arachne_delivery::RangeQuery {
6919 workspace: workspace.id(),
6920 author: workspace.member().unwrap().id(),
6921 epoch: workspace.epoch(),
6922 policy_revision: 17,
6923 after: 1,
6924 through: 2,
6925 topics: BTreeSet::from([Topic::new("streams/sample").unwrap()]),
6926 },
6927 )
6928 };
6929 assert_eq!(
6932 call(
6933 joiner,
6934 json!({"op":"discover_recovery_cutoff", "peer":peer,
6935 "revision":17, "topics":["streams/sample"]})
6936 )
6937 .unwrap_err(),
6938 "UnknownWorkspace"
6939 );
6940 assert_eq!(
6941 call(admin, json!({"op":"poll_admission"})).unwrap(),
6942 Value::Null
6943 );
6944 call(
6945 joiner,
6946 json!({"op":"install_member_policy", "revision":17,
6947 "topics":["streams/sample"]}),
6948 )
6949 .unwrap();
6950 let poll_recovery = || {
6951 let deadline = std::time::Instant::now() + Duration::from_secs(4);
6952 loop {
6953 let result = call(admin, json!({"op":"poll_admission"})).unwrap();
6954 if result != Value::Null {
6955 assert_eq!(result["state"], "recovery_replied");
6956 break;
6957 }
6958 assert!(
6959 std::time::Instant::now() < deadline,
6960 "recovery dispatch deadline"
6961 );
6962 std::thread::sleep(Duration::from_millis(10));
6963 }
6964 };
6965 let poll_result = |handle, operation: &str| {
6966 let deadline = std::time::Instant::now() + Duration::from_secs(4);
6967 loop {
6968 let result = call(handle, json!({"op":operation}));
6969 if !matches!(&result, Ok(Value::Null)) {
6970 break result;
6971 }
6972 assert!(
6973 std::time::Instant::now() < deadline,
6974 "cutoff completion deadline"
6975 );
6976 std::thread::sleep(Duration::from_millis(10));
6977 }
6978 };
6979 let poll_cutoff = |handle| poll_result(handle, "poll_recovery_cutoff");
6980 let poll_range = |handle| poll_result(handle, "poll_recovery_range");
6981 let exchange = |encoded_query: Vec<u8>| {
6982 let worker = std::thread::spawn(move || {
6983 let shared = session(joiner).unwrap();
6984 let guard = shared.lock().unwrap();
6985 let owner = guard.as_ref().unwrap();
6986 owner.runtime.block_on(async {
6987 owner.node.add_address_hint(peer, address).await.unwrap();
6988 owner
6989 .node
6990 .request_control(peer, &encoded_query)
6991 .await
6992 .unwrap()
6993 })
6994 });
6995 poll_recovery();
6996 worker.join().unwrap()
6997 };
6998 let cutoff = arachne_delivery::wire::CutoffQuery {
6999 workspace: query.workspace,
7000 author: query.author,
7001 epoch: query.epoch,
7002 policy_revision: query.policy_revision,
7003 topics: query.topics.clone(),
7004 nonce: [61; 32],
7005 };
7006 let cutoff_reply = exchange(cutoff.to_wire().unwrap());
7007 let discover_request = json!({"op":"discover_recovery_cutoff", "peer":peer,
7008 "revision":17, "topics":["streams/sample"]});
7009 let pending = call(joiner, discover_request.clone()).unwrap();
7011 assert_eq!(pending["state"], "recovery_cutoff_pending");
7012 assert!(call(joiner, discover_request.clone()).is_err());
7013 poll_recovery();
7014 let observed = poll_cutoff(joiner).unwrap();
7015 assert_eq!(
7016 call(joiner, json!({"op":"poll_recovery_cutoff"})).unwrap(),
7017 Value::Null
7018 );
7019 assert_eq!(observed["state"], "recovery_cutoff_observed");
7020 assert_eq!(observed["head"], 3);
7021 assert_eq!(observed["author"], created["member"]["id"]);
7022 assert_eq!(observed["accepted_through"], 0);
7023 assert_eq!(observed["accepted_progress"], false);
7024 assert!(observed.get("snapshot").is_none() && observed.get("nonce").is_none());
7025 let (joiner_peer, joiner_address) = {
7028 let shared = session(joiner).unwrap();
7029 let guard = shared.lock().unwrap();
7030 let owner = guard.as_ref().unwrap();
7031 (
7032 owner.node.id(),
7033 std::net::SocketAddr::from(([127, 0, 0, 1], owner.node.address().port())),
7034 )
7035 };
7036 call(
7037 admin,
7038 json!({"op":"add_address_hint", "peer":joiner_peer,
7039 "address":joiner_address.to_string()}),
7040 )
7041 .unwrap();
7042 let reverse_request = json!({"op":"discover_recovery_cutoff", "peer":joiner_peer,
7043 "revision":17, "topics":["streams/sample"]});
7044 assert_eq!(
7045 call(joiner, discover_request.clone()).unwrap()["state"],
7046 "recovery_cutoff_pending"
7047 );
7048 assert_eq!(
7049 call(admin, reverse_request.clone()).unwrap()["state"],
7050 "recovery_cutoff_pending"
7051 );
7052 let deadline = std::time::Instant::now() + Duration::from_secs(4);
7053 let mut completed = BTreeMap::new();
7054 while completed.len() < 2 {
7055 for handle in [admin, joiner] {
7056 call(handle, json!({"op":"poll_admission"})).unwrap();
7057 let result = call(handle, json!({"op":"poll_recovery_cutoff"})).unwrap();
7058 if result != Value::Null {
7059 assert_eq!(result["state"], "recovery_cutoff_observed");
7060 assert!(completed.insert(handle, result).is_none());
7061 }
7062 }
7063 assert!(
7064 std::time::Instant::now() < deadline,
7065 "crossed cutoff deadline"
7066 );
7067 std::thread::sleep(Duration::from_millis(10));
7068 }
7069 assert_eq!(completed[&admin]["head"], 1);
7070 assert_eq!(completed[&joiner]["head"], 3);
7071 for invalid in [
7072 json!({"op":"discover_recovery_cutoff", "peer":vec![99;32], "revision":17, "topics":["streams/sample"]}),
7073 json!({"op":"discover_recovery_cutoff", "peer":peer, "revision":16, "topics":["streams/sample"]}),
7074 json!({"op":"discover_recovery_cutoff", "peer":peer, "revision":17, "topics":["streams/other"]}),
7075 json!({"op":"discover_recovery_cutoff", "peer":peer, "revision":17, "topics":["streams/sample","streams/sample"]}),
7076 json!({"op":"discover_recovery_cutoff", "peer":peer, "revision":17, "topics":[]}),
7077 json!({"op":"discover_recovery_cutoff", "peer":peer, "revision":17, "topics":["streams/sample"], "nonce":vec![1;32]}),
7078 ] {
7079 assert!(call(joiner, invalid).is_err());
7080 }
7081 assert_eq!(
7082 call(admin, json!({"op":"poll_admission"})).unwrap(),
7083 Value::Null
7084 );
7085
7086 {
7087 let shared = session(joiner).unwrap();
7088 let guard = shared.lock().unwrap();
7089 let reader = guard.as_ref().unwrap().workspace.as_ref().unwrap();
7090 assert_eq!(
7091 arachne_delivery::wire::verify_cutoff_reply(reader, &cutoff, &cutoff_reply)
7092 .unwrap(),
7093 Some(3)
7094 );
7095 let mut next_request = cutoff.clone();
7096 next_request.nonce[0] ^= 1;
7097 assert!(
7098 arachne_delivery::wire::verify_cutoff_reply(reader, &next_request, &cutoff_reply)
7099 .is_err()
7100 );
7101 }
7102 assert_eq!(
7103 exchange(b"DFCQ\x02".to_vec()),
7104 arachne_delivery::wire::denied_reply()
7105 );
7106 let reply = exchange(query.to_wire().unwrap());
7107 {
7108 let shared = session(joiner).unwrap();
7109 let guard = shared.lock().unwrap();
7110 let reader = guard.as_ref().unwrap().workspace.as_ref().unwrap();
7111 let arachne_delivery::wire::RangeReply::Offered(range) =
7112 arachne_delivery::wire::verify_reply(reader, &query, &reply).unwrap()
7113 else {
7114 panic!("expected wire recovery offer")
7115 };
7116 assert_eq!(range.packets().len(), 1);
7117 assert_eq!(range.packets()[0].ciphertext, retained);
7118 assert_eq!(range.packets()[0].context.id, [12; 16]);
7119 }
7120 let fetch_request = json!({"op":"fetch_recovery_range", "peer":peer, "revision":17,
7121 "topics":["streams/sample"], "after":1, "through":2});
7122 assert_eq!(
7123 call(joiner, fetch_request.clone()).unwrap()["state"],
7124 "recovery_range_pending"
7125 );
7126 assert!(call(joiner, fetch_request.clone()).is_err());
7127 assert!(call(joiner, discover_request.clone()).is_err());
7128 poll_recovery();
7129 let ready = poll_range(joiner).unwrap();
7130 assert_eq!(ready["state"], "recovery_range_ready");
7131 assert_eq!(ready["packet_count"], 1);
7132 assert_eq!(ready["accepted_progress"], false);
7133 assert!(ready.get("payload").is_none() && ready.get("snapshot").is_none());
7134 {
7135 let shared = session(joiner).unwrap();
7136 let guard = shared.lock().unwrap();
7137 let owner = guard.as_ref().unwrap();
7138 let ready = owner.ready_range.as_ref().unwrap();
7139 let arachne_delivery::wire::RangeReply::Offered(range) =
7140 arachne_delivery::wire::verify_reply(
7141 owner.workspace.as_ref().unwrap(),
7142 &ready.query,
7143 &ready.reply,
7144 )
7145 .unwrap()
7146 else {
7147 panic!("expected retained verified range")
7148 };
7149 assert_eq!(range.packets()[0].ciphertext, retained);
7150 assert_eq!(range.packets()[0].context.id, [12; 16]);
7151 }
7152 assert_eq!(
7153 call(joiner, json!({"op":"poll_recovery_range"})).unwrap(),
7154 Value::Null
7155 );
7156 assert!(call(joiner, fetch_request.clone()).is_err());
7157 call(joiner, json!({"op":"cancel_recovery_range"})).unwrap();
7158 for invalid in [
7159 json!({"op":"fetch_recovery_range", "peer":peer, "revision":17,"topics":["streams/sample"],"after":2,"through":2}),
7160 json!({"op":"fetch_recovery_range", "peer":peer, "revision":16,"topics":["streams/sample"],"after":1,"through":2}),
7161 json!({"op":"fetch_recovery_range", "peer":vec![99;32], "revision":17,"topics":["streams/sample"],"after":1,"through":2}),
7162 json!({"op":"fetch_recovery_range", "peer":peer, "revision":17,"topics":["streams/sample","streams/sample"],"after":1,"through":2}),
7163 ] {
7164 assert!(call(joiner, invalid).is_err());
7165 }
7166 assert_eq!(
7167 call(admin, json!({"op":"poll_admission"})).unwrap(),
7168 Value::Null
7169 );
7170 let mut stale = arachne_delivery::RangeQuery::from_wire(&query.to_wire().unwrap()).unwrap();
7171 stale.policy_revision = 16;
7172 assert_eq!(
7173 exchange(stale.to_wire().unwrap()),
7174 arachne_delivery::wire::denied_reply()
7175 );
7176 assert_eq!(
7177 exchange(b"DFRQ\x02".to_vec()),
7178 arachne_delivery::wire::denied_reply()
7179 );
7180 call(
7181 admin,
7182 json!({"op":"install_member_policy","revision":18,"topics":["streams/other"]}),
7183 )
7184 .unwrap();
7185 assert_eq!(
7186 exchange(query.to_wire().unwrap()),
7187 arachne_delivery::wire::denied_reply()
7188 );
7189 assert_eq!(
7190 exchange(cutoff.to_wire().unwrap()),
7191 arachne_delivery::wire::denied_reply()
7192 );
7193 let mut revoked_cutoff = cutoff.clone();
7194 revoked_cutoff.policy_revision = 18;
7195 assert_eq!(
7196 exchange(revoked_cutoff.to_wire().unwrap()),
7197 arachne_delivery::wire::denied_reply()
7198 );
7199 call(joiner, fetch_request.clone()).unwrap();
7200 poll_recovery();
7201 let rejected = poll_range(joiner).unwrap();
7202 assert_eq!(rejected["state"], "recovery_range_rejected");
7203 assert_eq!(rejected["reason"], "Denied");
7204 assert!(
7205 session(joiner)
7206 .unwrap()
7207 .lock()
7208 .unwrap()
7209 .as_ref()
7210 .unwrap()
7211 .ready_range
7212 .is_none()
7213 );
7214 call(joiner, discover_request.clone()).unwrap();
7216 poll_recovery();
7217 let denied = poll_cutoff(joiner).unwrap();
7218 assert_eq!(denied["state"], "recovery_cutoff_denied");
7219 assert!(denied.get("head").is_none());
7220 stale.policy_revision = 18;
7221 assert_eq!(
7222 exchange(stale.to_wire().unwrap()),
7223 arachne_delivery::wire::denied_reply()
7224 );
7225 stale.topics = BTreeSet::from([Topic::new("streams/other").unwrap()]);
7228 let empty_reply = exchange(stale.to_wire().unwrap());
7229 {
7230 let shared = session(joiner).unwrap();
7231 let guard = shared.lock().unwrap();
7232 let reader = guard.as_ref().unwrap().workspace.as_ref().unwrap();
7233 let arachne_delivery::wire::RangeReply::Offered(range) =
7234 arachne_delivery::wire::verify_reply(reader, &stale, &empty_reply).unwrap()
7235 else {
7236 panic!("expected signed empty recovery offer")
7237 };
7238 assert!(range.packets().is_empty());
7239 }
7240 assert_eq!(
7241 call(admin, json!({"op":"poll_admission"})).unwrap(),
7242 Value::Null
7243 );
7244 call(joiner, discover_request.clone()).unwrap();
7247 poll_recovery();
7248 call(
7249 joiner,
7250 json!({"op":"install_member_policy", "revision":18, "topics":["streams/other"]}),
7251 )
7252 .unwrap();
7253 assert!(poll_cutoff(joiner).is_err());
7254 assert_eq!(
7255 call(joiner, json!({"op":"poll_recovery_cutoff"})).unwrap(),
7256 Value::Null
7257 );
7258 let empty_request = json!({"op":"fetch_recovery_range", "peer":peer, "revision":18,
7259 "topics":["streams/other"], "after":1, "through":2});
7260 call(joiner, empty_request.clone()).unwrap();
7261 poll_recovery();
7262 let empty = poll_range(joiner).unwrap();
7263 assert_eq!(empty["state"], "recovery_range_ready");
7264 assert_eq!(empty["packet_count"], 0);
7265 call(joiner, json!({"op":"cancel_recovery_range"})).unwrap();
7266 call(joiner, empty_request.clone()).unwrap();
7267 poll_recovery();
7268 call(
7269 joiner,
7270 json!({"op":"install_member_policy", "revision":19, "topics":["streams/other"]}),
7271 )
7272 .unwrap();
7273 assert!(poll_range(joiner).is_err());
7274 assert!(
7275 session(joiner)
7276 .unwrap()
7277 .lock()
7278 .unwrap()
7279 .as_ref()
7280 .unwrap()
7281 .ready_range
7282 .is_none()
7283 );
7284 let mut cancelled_request = empty_request;
7285 cancelled_request["revision"] = json!(19);
7286 call(joiner, cancelled_request).unwrap();
7287 let aborted_range = {
7288 let shared = session(joiner).unwrap();
7289 let guard = shared.lock().unwrap();
7290 guard
7291 .as_ref()
7292 .unwrap()
7293 .range
7294 .as_ref()
7295 .unwrap()
7296 .task
7297 .abort_handle()
7298 };
7299 call(joiner, json!({"op":"cancel_recovery_range"})).unwrap();
7300 let deadline = std::time::Instant::now() + Duration::from_secs(1);
7301 while !aborted_range.is_finished() {
7302 assert!(std::time::Instant::now() < deadline);
7303 std::thread::sleep(Duration::from_millis(5));
7304 }
7305 call(
7309 admin,
7310 json!({"op":"install_member_policy", "revision":19,
7311 "topics":["streams/other"]}),
7312 )
7313 .unwrap();
7314 for id in 90..97 {
7315 let missed = call(
7316 admin,
7317 json!({"op":"stage_network_publication", "revision":19,
7318 "topic":"streams/other", "id":vec![id;16], "payload":[id]}),
7319 )
7320 .unwrap();
7321 call(
7322 admin,
7323 json!({"op":"adopt_publication", "snapshot":missed["snapshot"]}),
7324 )
7325 .unwrap();
7326 }
7327 let head = {
7328 let shared = session(admin).unwrap();
7329 let guard = shared.lock().unwrap();
7330 guard.as_ref().unwrap().publisher.as_ref().unwrap().head()
7331 };
7332 let recover = json!({"op":"fetch_recovery_range", "peer":peer, "revision":19,
7333 "topics":["streams/other"], "after":0, "through":head});
7334 call(joiner, recover.clone()).unwrap();
7335 call(
7338 joiner,
7339 json!({"op":"subscribe", "workspace":created["workspace"],
7340 "revision":19, "topic":"streams/other"}),
7341 )
7342 .unwrap();
7343 let live = call(
7344 admin,
7345 json!({"op":"stage_network_publication", "revision":19,
7346 "topic":"streams/other", "id":vec![107;16], "payload":[8]}),
7347 )
7348 .unwrap();
7349 call(
7350 admin,
7351 json!({"op":"adopt_publication", "snapshot":live["snapshot"]}),
7352 )
7353 .unwrap();
7354 let deadline = std::time::Instant::now() + Duration::from_secs(2);
7355 loop {
7356 let shared = session(joiner).unwrap();
7357 if !shared.lock().unwrap().as_ref().unwrap().receiver.is_empty() {
7358 break;
7359 }
7360 assert!(
7361 std::time::Instant::now() < deadline,
7362 "newer live packet was not queued"
7363 );
7364 std::thread::sleep(Duration::from_millis(5));
7365 }
7366 assert!(
7367 call(joiner, json!({"op":"poll_protected"}))
7368 .unwrap()
7369 .is_null(),
7370 "live reception must wait for pending recovery"
7371 );
7372 assert_eq!(
7373 call(
7374 joiner,
7375 json!({"op":"stage_reception", "context":[], "ciphertext":[]})
7376 )
7377 .unwrap_err(),
7378 "recovery must finish before live reception"
7379 );
7380
7381 poll_recovery();
7382 assert_eq!(poll_range(joiner).unwrap()["packet_count"], 7);
7383 assert_eq!(
7384 call(joiner, json!({"op":"poll_recovered_publication"})).unwrap(),
7385 Value::Null
7386 );
7387 assert!(
7388 call(joiner, json!({"op":"poll_protected"}))
7389 .unwrap()
7390 .is_null(),
7391 "live reception must wait for ready recovery"
7392 );
7393 let [metadata, recovery_snapshot] =
7394 execute_stored(joiner, br#"{"op":"stage_recovery_range"}"#, &[]).unwrap();
7395 let metadata: Value = serde_json::from_slice(&metadata).unwrap();
7396 assert_eq!(metadata["state"], "awaiting_recovery_save");
7397 assert_eq!(metadata["publication_count"], 7);
7398 assert!(metadata.get("payload").is_none() && metadata.get("snapshot").is_none());
7399 assert!(call(joiner, json!({"op":"poll_recovered_publication"})).is_err());
7400 assert!(
7401 execute_stored(joiner, br#"{"op":"adopt_reception"}"#, &recovery_snapshot).is_err()
7402 );
7403 let mut corrupt = recovery_snapshot.clone();
7404 *corrupt.last_mut().unwrap() ^= 1;
7405 assert!(execute_stored(joiner, br#"{"op":"adopt_recovery"}"#, &corrupt).is_err());
7406 {
7407 let shared = session(joiner).unwrap();
7408 let guard = shared.lock().unwrap();
7409 assert!(guard.as_ref().unwrap().received.is_none());
7410 }
7411 let adopted =
7412 execute_stored(joiner, br#"{"op":"adopt_recovery"}"#, &recovery_snapshot).unwrap();
7413 assert_eq!(
7414 serde_json::from_slice::<Value>(&adopted[0]).unwrap()["publication_count"],
7415 7
7416 );
7417 assert!(call(joiner, json!({"op":"seal_workspace"})).is_err());
7418 for id in 90..97 {
7419 let recovered = call(joiner, json!({"op":"poll_recovered_publication"})).unwrap();
7420 assert_eq!(recovered["payload"], json!([id]));
7421 assert_eq!(recovered["id"], json!(vec![id; 16]));
7422 assert_eq!(recovered["topic"], "streams/other");
7423 }
7424 assert_eq!(
7425 call(joiner, json!({"op":"poll_recovered_publication"})).unwrap(),
7426 Value::Null
7427 );
7428 assert!(execute_stored(joiner, br#"{"op":"adopt_recovery"}"#, &recovery_snapshot).is_err());
7429 call(joiner, recover.clone()).unwrap();
7430 poll_recovery();
7431 poll_range(joiner).unwrap();
7432 assert_eq!(
7433 call(joiner, json!({"op":"stage_recovery_range"})).unwrap()["state"],
7434 "recovery_already_covered"
7435 );
7436 let outgoing = call(
7438 joiner,
7439 json!({"op":"stage_network_publication", "revision":19,
7440 "topic":"streams/other", "id":vec![91;16], "payload":[7]}),
7441 )
7442 .unwrap();
7443 call(
7444 joiner,
7445 json!({"op":"adopt_publication", "snapshot":outgoing["snapshot"]}),
7446 )
7447 .unwrap();
7448 let live = poll_result(joiner, "poll_protected").unwrap();
7449 assert!(live.get("payload").is_none());
7450 let live = call(
7451 joiner,
7452 json!({"op":"adopt_reception", "snapshot":live["snapshot"]}),
7453 )
7454 .unwrap();
7455 assert_eq!(live["payload"], json!([8]));
7456 let mut continuation = recover;
7457 continuation["after"] = json!(head);
7458 continuation["through"] = json!(head + 1);
7459 call(joiner, continuation).unwrap();
7460 poll_recovery();
7461 poll_range(joiner).unwrap();
7462 let [metadata, snapshot] =
7463 execute_stored(joiner, br#"{"op":"stage_recovery_range"}"#, &[]).unwrap();
7464 let metadata: Value = serde_json::from_slice(&metadata).unwrap();
7465 assert_eq!(metadata["publication_count"], 0);
7466 assert_eq!(metadata["already_received"], 1);
7467 execute_stored(joiner, br#"{"op":"adopt_recovery"}"#, &snapshot).unwrap();
7468 let [_, durable] = execute_stored(joiner, br#"{"op":"seal_workspace"}"#, &[]).unwrap();
7469 let journal_before = {
7470 let shared = session(joiner).unwrap();
7471 let guard = shared.lock().unwrap();
7472 guard
7473 .as_ref()
7474 .unwrap()
7475 .received
7476 .as_ref()
7477 .unwrap()
7478 .snapshot()
7479 };
7480 let mut pending_on_close = discover_request;
7481 pending_on_close["revision"] = json!(19);
7482 pending_on_close["topics"] = json!(["streams/other"]);
7483 call(joiner, pending_on_close).unwrap();
7484 let task = {
7485 let shared = session(joiner).unwrap();
7486 let guard = shared.lock().unwrap();
7487 guard
7488 .as_ref()
7489 .unwrap()
7490 .cutoff
7491 .as_ref()
7492 .unwrap()
7493 .task
7494 .abort_handle()
7495 };
7496 close(joiner).unwrap();
7497 assert!(task.is_finished());
7498 let restored = create(Some(&[42; 32])).unwrap();
7499 execute_stored(
7500 restored,
7501 &serde_json::to_vec(&json!({"op":"restore_workspace",
7502 "workspace":created["workspace"]}))
7503 .unwrap(),
7504 &durable,
7505 )
7506 .unwrap();
7507 {
7508 let shared = session(restored).unwrap();
7509 let guard = shared.lock().unwrap();
7510 let session = guard.as_ref().unwrap();
7511 assert_eq!(
7512 session.received.as_ref().unwrap().snapshot(),
7513 journal_before
7514 );
7515 let selection = BTreeSet::from([Topic::new("streams/other").unwrap()]);
7516 let author = session
7517 .workspace
7518 .as_ref()
7519 .unwrap()
7520 .member_id_for_endpoint(peer)
7521 .unwrap();
7522 assert_eq!(
7523 session
7524 .received
7525 .as_ref()
7526 .unwrap()
7527 .progress(author, &selection),
7528 Some(head + 1)
7529 );
7530 assert!(session.publisher.as_ref().unwrap().head() > 0);
7531 }
7532 assert_eq!(
7533 call(restored, json!({"op":"poll_recovered_publication"})).unwrap(),
7534 Value::Null
7535 );
7536 for handle in [admin, restored] {
7539 let staged = call(handle, json!({"op":"enable_object_delivery"})).unwrap();
7540 assert!(staged.get("payload").is_none());
7541 assert!(call(handle, json!({"op":"poll_pending_object"})).is_err());
7542 call(
7543 handle,
7544 json!({"op":"adopt_reception", "snapshot":staged["snapshot"]}),
7545 )
7546 .unwrap();
7547 if handle == admin {
7548 call(
7549 handle,
7550 json!({"op":"install_member_policy", "revision":20,"topics":["streams/objects"]}),
7551 )
7552 .unwrap();
7553 }
7554 }
7555 let send_object = |number: u8| {
7556 let staged = call(
7557 admin,
7558 json!({"op":"stage_network_publication", "revision":20,
7559 "topic":"streams/objects", "id":vec![number;16], "payload":[number]}),
7560 )
7561 .unwrap();
7562 call(
7563 admin,
7564 json!({"op":"adopt_publication", "snapshot":staged["snapshot"]}),
7565 )
7566 .unwrap()
7567 };
7568 let current = call(
7569 admin,
7570 json!({"op":"stage_network_publication", "revision":20,
7571 "topic":"streams/objects", "id":vec![100;16], "payload":[100],
7572 "current":{"selector":vec![7;32], "replacement_key":vec![8;32],
7573 "expires_at":u64::MAX}}),
7574 )
7575 .unwrap();
7576 call(
7577 admin,
7578 json!({"op":"adopt_publication", "snapshot":current["snapshot"]}),
7579 )
7580 .unwrap();
7581 let missed = send_object(101);
7582 let after = missed["sequence"].as_u64().unwrap() - 1;
7583 let connect_objects = |handle| {
7584 call(
7585 handle,
7586 json!({"op":"add_address_hint","peer":peer,"address":address.to_string()}),
7587 )
7588 .unwrap();
7589 call(
7590 handle,
7591 json!({"op":"install_member_policy", "revision":20,"topics":["streams/objects"]}),
7592 )
7593 .unwrap();
7594 call(handle, json!({"op":"subscribe", "workspace":created["workspace"],"revision":20,"topic":"streams/objects"})).unwrap();
7595 };
7596 connect_objects(restored);
7597 let live = send_object(102);
7598 assert_eq!(live["admission"]["queued"], true);
7599 let staged = poll_result(restored, "poll_protected").unwrap();
7600 assert!(staged.get("payload").is_none());
7601 call(
7602 restored,
7603 json!({"op":"adopt_reception", "snapshot":staged["snapshot"]}),
7604 )
7605 .unwrap();
7606 let pending = call(restored, json!({"op":"poll_pending_object"})).unwrap();
7607 assert_eq!(pending["payload"], json!([102]));
7608 assert_eq!(
7609 call(restored, json!({"op":"poll_pending_object"})).unwrap(),
7610 pending
7611 );
7612 let [_, pending_snapshot] =
7613 execute_stored(restored, br#"{"op":"seal_workspace"}"#, &[]).unwrap();
7614 close(restored).unwrap();
7615 let restored = create(Some(&[42; 32])).unwrap();
7616 execute_stored(
7617 restored,
7618 &serde_json::to_vec(
7619 &json!({"op":"restore_workspace","workspace":created["workspace"]}),
7620 )
7621 .unwrap(),
7622 &pending_snapshot,
7623 )
7624 .unwrap();
7625 assert_eq!(
7626 call(restored, json!({"op":"poll_pending_object"})).unwrap(),
7627 pending
7628 );
7629 {
7630 let shared = session(restored).unwrap();
7631 let guard = shared.lock().unwrap();
7632 assert_eq!(
7633 guard
7634 .as_ref()
7635 .unwrap()
7636 .received
7637 .as_ref()
7638 .unwrap()
7639 .snapshot(),
7640 journal_before
7641 );
7642 }
7643 let ack = |handle, pending: &Value| {
7644 let request = json!({"op":"stage_object_acknowledgement", "member":pending["member"],
7645 "topic":pending["topic"], "id":pending["id"], "counter":pending["counter"]});
7646 let mut wrong = request.clone();
7647 wrong["id"] = json!(vec![0; 16]);
7648 assert!(call(handle, wrong).is_err());
7649 let staged = call(handle, request).unwrap();
7650 let snapshot: Vec<u8> = serde_json::from_value(staged["snapshot"].clone()).unwrap();
7651 assert_eq!(
7652 execute_stored(handle, br#"{"op":"adopt_publication"}"#, &snapshot).unwrap_err(),
7653 "wrong adoption lifecycle phase"
7654 );
7655 execute_stored(handle, br#"{"op":"adopt_reception"}"#, &snapshot).unwrap();
7656 };
7657 ack(restored, &pending);
7658 assert_eq!(
7659 call(restored, json!({"op":"poll_pending_object"})).unwrap(),
7660 Value::Null
7661 );
7662 connect_objects(restored);
7663 let next = call(restored, json!({"op":"next_membership_peer"})).unwrap();
7664 assert_eq!(next["peer"], json!(peer));
7665 let authority = next["member"].clone();
7666 call(
7667 restored,
7668 json!({"op":"fetch_current_view", "authority":authority,
7669 "revision":20, "topic":"streams/objects", "selector":vec![7;32]}),
7670 )
7671 .unwrap();
7672 let deadline = std::time::Instant::now() + Duration::from_secs(3);
7673 let ready = loop {
7674 let response = call(admin, json!({"op":"poll_admission"})).unwrap();
7675 assert!(response.is_null() || response["state"] == "current_view_replied");
7676 let response = call(restored, json!({"op":"poll_current_view"})).unwrap();
7677 if !response.is_null() {
7678 break response;
7679 }
7680 assert!(
7681 std::time::Instant::now() < deadline,
7682 "current-view control deadline"
7683 );
7684 std::thread::sleep(Duration::from_millis(10));
7685 };
7686 assert_eq!(ready["state"], "current_view_ready");
7687 assert_eq!(ready["automatic_source"], true);
7688 assert_eq!(ready["cut"], 1);
7689 assert_eq!(ready["value_count"], 1);
7690 let staged = call(restored, json!({"op":"stage_current_view"})).unwrap();
7691 assert_eq!(staged["state"], "awaiting_current_view_save");
7692 assert_eq!(staged["pending"], 1);
7693 assert!(call(restored, json!({"op":"poll_pending_object"})).is_err());
7694 call(
7695 restored,
7696 json!({"op":"adopt_current_view", "snapshot":staged["snapshot"]}),
7697 )
7698 .unwrap();
7699 let current = call(restored, json!({"op":"poll_pending_object"})).unwrap();
7700 assert_eq!(current["payload"], json!([100]));
7701 assert_eq!(current["current"]["replacement_key"], json!(vec![8; 32]));
7702 ack(restored, ¤t);
7703 call(
7704 restored,
7705 json!({"op":"fetch_recovery_range", "peer":peer, "revision":20,
7706 "topics":["streams/objects"], "after":after, "through":live["sequence"]}),
7707 )
7708 .unwrap();
7709 poll_recovery();
7710 poll_range(restored).unwrap();
7711 let [metadata, snapshot] =
7712 execute_stored(restored, br#"{"op":"stage_recovery_range"}"#, &[]).unwrap();
7713 let metadata: Value = serde_json::from_slice(&metadata).unwrap();
7714 assert_eq!(metadata["publication_count"], 1); execute_stored(restored, br#"{"op":"adopt_recovery"}"#, &snapshot).unwrap();
7716 let recovered = call(restored, json!({"op":"poll_pending_object"})).unwrap();
7717 assert_eq!(recovered["payload"], json!([101]));
7718 ack(restored, &recovered);
7719 let [_, snapshot] = execute_stored(restored, br#"{"op":"seal_workspace"}"#, &[]).unwrap();
7720 close(restored).unwrap();
7721 let restored = create(Some(&[42; 32])).unwrap();
7722 execute_stored(
7723 restored,
7724 &serde_json::to_vec(
7725 &json!({"op":"restore_workspace","workspace":created["workspace"]}),
7726 )
7727 .unwrap(),
7728 &snapshot,
7729 )
7730 .unwrap();
7731 assert_eq!(
7732 call(restored, json!({"op":"poll_pending_object"})).unwrap(),
7733 Value::Null
7734 );
7735 connect_objects(restored);
7736 call(
7737 restored,
7738 json!({"op":"fetch_recovery_range", "peer":peer,"revision":20,
7739 "topics":["streams/objects"],"after":after,"through":live["sequence"]}),
7740 )
7741 .unwrap();
7742 poll_recovery();
7743 poll_range(restored).unwrap();
7744 assert_eq!(
7745 call(restored, json!({"op":"stage_recovery_range"})).unwrap()["state"],
7746 "recovery_no_new_objects"
7747 );
7748 let current_live = call(
7749 admin,
7750 json!({"op":"stage_network_publication", "revision":20,
7751 "topic":"streams/objects", "id":vec![109;16], "payload":[109],
7752 "current":{"selector":vec![7;32], "replacement_key":vec![8;32],
7753 "expires_at":u64::MAX}}),
7754 )
7755 .unwrap();
7756 call(
7757 admin,
7758 json!({"op":"adopt_publication", "snapshot":current_live["snapshot"]}),
7759 )
7760 .unwrap();
7761 let staged = poll_result(restored, "poll_protected").unwrap();
7762 call(
7763 restored,
7764 json!({"op":"adopt_reception", "snapshot":staged["snapshot"]}),
7765 )
7766 .unwrap();
7767 let current_pending = call(restored, json!({"op":"poll_pending_object"})).unwrap();
7768 assert_eq!(current_pending["payload"], json!([109]));
7769 assert_eq!(current_pending["current"]["selector"], json!(vec![7; 32]));
7770 assert_eq!(
7771 current_pending["current"]["replacement_key"],
7772 json!(vec![8; 32])
7773 );
7774 ack(restored, ¤t_pending);
7775 for number in 110..150 {
7776 send_object(number);
7777 }
7778 call(restored, json!({"op":"discover_recovery_cutoff", "peer":peer,"revision":20,"topics":["streams/objects"]})).unwrap();
7779 poll_recovery();
7780 let window = poll_cutoff(restored).unwrap();
7781 assert!(window["retained_after"].as_u64().unwrap() > after);
7782 assert_eq!(window["accepted_through"], 0);
7783 call(restored, json!({"op":"fetch_recovery_range", "peer":peer,"revision":20,
7784 "topics":["streams/objects"],"after":window["retained_after"],"through":window["head"]})).unwrap();
7785 poll_recovery();
7786 poll_range(restored).unwrap();
7787 let [metadata, snapshot] =
7788 execute_stored(restored, br#"{"op":"stage_recovery_range"}"#, &[]).unwrap();
7789 assert_eq!(
7790 serde_json::from_slice::<Value>(&metadata).unwrap()["publication_count"],
7791 32
7792 );
7793 execute_stored(restored, br#"{"op":"adopt_recovery"}"#, &snapshot).unwrap();
7794 let service = create(Some(&[43; 32])).unwrap();
7795 let invite = call(admin, json!({"op":"issue_invitation"})).unwrap();
7796 let pending_service = call(
7797 service,
7798 json!({"op":"begin_join", "invitation":invite["invitation"],
7799 "checkpoint":invite["checkpoint"], "display_name":"Air traffic feed"}),
7800 )
7801 .unwrap();
7802 let admission = json!({"op":"stage_admission", "authenticated_endpoint":pending_service["endpoint"],
7803 "request":pending_service["admission_request"]});
7804 let [_, saved] =
7805 execute_stored(admin, &serde_json::to_vec(&admission).unwrap(), &[]).unwrap();
7806 execute_stored(admin, br#"{"op":"adopt_admission"}"#, &saved).unwrap();
7807 let response = call(
7808 admin,
7809 json!({"op":"retained_admission", "authenticated_endpoint":pending_service["endpoint"],
7810 "request":pending_service["admission_request"]}),
7811 )
7812 .unwrap();
7813 let step = json!({"commit":response["commit"], "authorization":response["authorization"]});
7814 call(
7815 restored,
7816 json!({"op":"fetch_membership_update", "peer":peer}),
7817 )
7818 .unwrap();
7819 let deadline = std::time::Instant::now() + Duration::from_secs(10);
7820 let fetched = loop {
7821 call(admin, json!({"op":"poll_admission"})).unwrap();
7822 let result = call(restored, json!({"op":"poll_membership_update"})).unwrap();
7823 if result != Value::Null {
7824 break result;
7825 }
7826 assert!(
7827 std::time::Instant::now() < deadline,
7828 "membership query deadline"
7829 );
7830 std::thread::sleep(Duration::from_millis(10));
7831 };
7832 assert_eq!(fetched["state"], "membership_update_available");
7833 assert_eq!(fetched["step"], step);
7834 assert!(fetched.get("welcome").is_none());
7835 let update = json!({"op":"stage_admission_update", "step":fetched["step"]});
7836 assert_eq!(
7837 call(restored, update.clone()).unwrap_err(),
7838 "pending application delivery must be acknowledged before membership update"
7839 );
7840 assert_eq!(
7841 call(restored, admission).unwrap_err(),
7842 "pending application delivery must be acknowledged before membership update"
7843 );
7844 for number in 118..150 {
7845 let pending = call(restored, json!({"op":"poll_pending_object"})).unwrap();
7846 assert_eq!(pending["payload"], json!([number]));
7847 ack(restored, &pending);
7848 }
7849 println!(
7850 "NATIVE_OBJECT_INBOX legacy_preserved=true live_pending_restart=true acknowledgement_restart=true missed_recovery=true adapter_callback=not_exercised"
7851 );
7852 let [_, saved] =
7853 execute_stored(restored, &serde_json::to_vec(&update).unwrap(), &[]).unwrap();
7854 let mut altered = saved.clone();
7855 *altered.last_mut().unwrap() ^= 1;
7856 assert!(execute_stored(restored, br#"{"op":"adopt_admission"}"#, &altered).is_err());
7857 assert!(execute_stored(restored, br#"{"op":"adopt_reception"}"#, &saved).is_err());
7858 close(restored).unwrap(); let restored = create(Some(&[42; 32])).unwrap();
7860 let [metadata, _] = execute_stored(
7861 restored,
7862 &serde_json::to_vec(
7863 &json!({"op":"restore_workspace", "workspace":created["workspace"]}),
7864 )
7865 .unwrap(),
7866 &saved,
7867 )
7868 .unwrap();
7869 assert_eq!(
7870 serde_json::from_slice::<Value>(&metadata).unwrap()["members"],
7871 3
7872 );
7873 assert!(call(restored, update).is_err()); let [_, saved] = execute_stored(
7875 service,
7876 &serde_json::to_vec(
7877 &json!({"op":"stage_join", "commits":[step], "welcome":response["welcome"]}),
7878 )
7879 .unwrap(),
7880 &[],
7881 )
7882 .unwrap();
7883 let [metadata, _] = execute_stored(service, br#"{"op":"adopt_join"}"#, &saved).unwrap();
7884 assert_eq!(
7885 serde_json::from_slice::<Value>(&metadata).unwrap()["members"],
7886 3
7887 );
7888 close(service).unwrap();
7889 close(restored).unwrap();
7890 close(admin).unwrap();
7891 }
7892}