1#![allow(clippy::large_futures)]
9
10use std::collections::BTreeSet;
11use std::sync::Arc;
12use std::time::Duration;
13
14use meerkat_core::agent::CommsRuntime;
15#[allow(unused_imports)]
16use meerkat_core::comms::{
17 CommsCommand, CommsTrustMutation, CommsTrustMutationAuthority, CommsTrustMutationResult,
18 PeerId, PeerRoute, SendError, TrustedPeerDescriptor,
19};
20#[cfg(test)]
21use meerkat_core::comms::{PeerAddress, SendReceipt};
22use meerkat_core::event::AgentEvent;
23use meerkat_core::interaction::{
24 InteractionContent, PeerIngressFact, PeerInputCandidate, PeerInputClass,
25};
26use meerkat_core::time_compat::Instant;
27use meerkat_core::types::SessionId;
28
29use meerkat_contracts::wire::WireMemberHistoryPageBody;
30use meerkat_contracts::wire::supervisor_bridge::{
31 BRIDGE_TURN_OUTCOME_ACK_MAX, BridgeAck, BridgeBindResponse, BridgeCapabilities, BridgeCommand,
32 BridgeDeliveryOutcome, BridgeDeliveryPayload, BridgeDeliveryRejectionCause,
33 BridgeDeliveryResponse, BridgeDestroyResponse, BridgeEventCursor, BridgeHardCancelPayload,
34 BridgeLiveControlledResponse, BridgeLiveOpenedResponse, BridgeMemberEventsPage,
35 BridgeMemberHistoryPage, BridgeMemberIncarnation, BridgeMemberRuntimeState,
36 BridgeMobPeerOverlayHandoff, BridgeObservationResponse, BridgeOutboundTaintTarget,
37 BridgeOutcomeTracking, BridgePeerConnectivity, BridgePeerIdentity, BridgePeerSpec,
38 BridgeProtocolVersion, BridgeRejectionCause, BridgeReply, BridgeRetireResponse,
39 BridgeSupervisorPayload, BridgeSupervisorRotationObservation,
40 BridgeSupervisorRotationOperationReceipt, BridgeSupervisorRotationPendingPhase,
41 BridgeSupervisorRotationRejectionCause, BridgeSupervisorRotationRejectionReceipt,
42 BridgeSupervisorRotationState, BridgeSupervisorRotationTargetReceipt,
43 BridgeTrackedInputCancelPayload, BridgeTrackedInputCancelResponse, SUPERVISOR_BRIDGE_INTENT,
44 SupervisorRotationOperationId, WireEventRow, canonicalize_bridge_address,
45 decode_bridge_command,
46};
47#[cfg(test)]
48use meerkat_contracts::wire::supervisor_bridge::{
49 BridgeHostRuntimeIncarnation, SUPERVISOR_BRIDGE_BOOTSTRAP_TOKEN_PARAM,
50 SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
51};
52
53use crate::comms_bridge::classified_interaction_to_runtime_input;
54#[cfg(test)]
55use crate::completion::CompletionOutcome;
56use crate::completion::interaction_terminal_event;
57use crate::identifiers::IdempotencyKey;
58#[cfg(test)]
59use crate::identifiers::LogicalRuntimeId;
60use crate::input::{
61 FlowStepInput, Input, InputDurability, InputHeader, InputOrigin, InputVisibility,
62 PeerConvention, PeerInput,
63};
64use crate::member_live::{MemberLiveError, MemberLiveHost};
65use crate::member_observation::{
66 DirectedTurnAdmission, DirectedTurnTracking, DirectedTurnWindow, MemberEventsPollRequest,
67 MemberEventsWindow, MemberHistoryWindow, MemberObservationCursor, MemberObservationError,
68 MemberObservationHost,
69};
70
71use crate::meerkat_machine::SupervisorBinding;
72use crate::meerkat_machine::{
73 DrainExitReason, GeneratedSupervisorBinding, GeneratedSupervisorRotationSubmit, MeerkatMachine,
74 SupervisorAuthorizeAdmission, SupervisorBindAdmission, SupervisorBindingStageError,
75 SupervisorRotationObservation, SupervisorRotationSubmission,
76};
77use crate::service_ext::SessionServiceRuntimeExt as _;
78use crate::tokio::sync::mpsc;
79use crate::traits::RuntimeControlPlane;
80
81pub const DEFAULT_IDLE_TIMEOUT: Duration = Duration::from_secs(300);
83const SUPERVISOR_RESPONSE_ROUTE_IO_TIMEOUT: Duration = Duration::from_secs(5);
84
85pub fn spawn_comms_drain(
92 adapter: Arc<MeerkatMachine>,
93 session_id: SessionId,
94 comms_runtime: Arc<dyn CommsRuntime>,
95 idle_timeout: Option<Duration>,
96) -> crate::tokio::task::JoinHandle<()> {
97 let timeout_dur = idle_timeout.unwrap_or(DEFAULT_IDLE_TIMEOUT);
98 let runtime_id = MeerkatMachine::logical_runtime_id(&session_id);
99
100 crate::tokio::spawn(async move {
101 if std::env::var_os("RKAT_TRACE_COMMS_DRAIN_BIND").is_some() {
102 tracing::info!(
103 %session_id,
104 comms_ptr = ?Arc::as_ptr(&comms_runtime),
105 "comms_drain task started"
106 );
107 }
108 let inbox_notify = comms_runtime.inbox_notify();
109 match adapter
110 .peer_ingress_runtime_is_current(&session_id, &comms_runtime)
111 .await
112 {
113 Ok(true) => {}
114 Ok(false) => {
115 tracing::warn!(%session_id, "stale comms drain refused startup");
116 return;
117 }
118 Err(error) => {
119 tracing::warn!(%session_id, %error, "comms drain lost session before startup");
120 return;
121 }
122 }
123 if let Err(error) = async {
124 adapter
125 .stage_local_endpoint_for_comms_runtime(&session_id, comms_runtime.as_ref())
126 .await
127 .map_err(|error| error.to_string())?;
128 hydrate_or_resume_supervisor_rotation_runtime(&adapter, &session_id, &comms_runtime)
129 .await
130 }
131 .await
132 {
133 tracing::warn!(%session_id, %error, "unable to hydrate supervisor authority on comms runtime");
134 }
135 loop {
141 let mut notified = std::pin::pin!(inbox_notify.notified());
151 notified.as_mut().enable();
152
153 let candidates = match comms_runtime.drain_classified_inbox_interactions().await {
161 Ok(candidates) => candidates,
162 Err(err) => {
163 tracing::error!(
167 session_id = %session_id,
168 error = %err,
169 "comms_drain: classified inbox drain failed; exiting via typed Failed terminal (not idle/dismiss)"
170 );
171 if let Err(notify_err) = adapter
172 .notify_comms_drain_exited(&session_id, DrainExitReason::Failed)
173 .await
174 {
175 tracing::error!(
180 session_id = %session_id,
181 error = %notify_err,
182 "comms_drain: NotifyDrainExited(Failed) rejected by machine authority"
183 );
184 adapter
185 .project_comms_drain_failed_safety_net(&session_id)
186 .await;
187 }
188 return;
189 }
190 };
191 if candidates.is_empty() {
192 if crate::tokio::time::timeout(timeout_dur, notified.as_mut())
201 .await
202 .is_err()
203 {
204 tracing::info!("comms_drain: idle timeout expired, stopping");
205 if let Err(notify_err) = adapter
206 .notify_comms_drain_exited(&session_id, DrainExitReason::IdleTimeout)
207 .await
208 {
209 tracing::error!(
213 session_id = %session_id,
214 error = %notify_err,
215 "comms_drain: NotifyDrainExited(IdleTimeout) rejected by machine authority"
216 );
217 adapter
218 .project_comms_drain_failed_safety_net(&session_id)
219 .await;
220 }
221 return;
222 }
223 continue;
224 }
225
226 for candidate in candidates {
228 if try_handle_supervisor_bridge_command(
229 &adapter,
230 &session_id,
231 &comms_runtime,
232 &candidate,
233 )
234 .await
235 {
236 continue;
237 }
238 let candidate_class = candidate.class();
239 match candidate_class {
240 PeerInputClass::Ack => {
241 }
243 PeerInputClass::PeerLifecycleAdded
244 | PeerInputClass::PeerLifecycleRetired
245 | PeerInputClass::PeerLifecycleUnwired => {
246 tracing::debug!(
253 session_id = %session_id,
254 class = ?candidate_class,
255 from_peer_id = ?candidate.from_peer_id(),
256 lifecycle_peer = ?candidate.lifecycle_peer,
257 "comms_drain: consumed silent peer lifecycle notice"
258 );
259 }
260 PeerInputClass::ResponseProgress | PeerInputClass::ResponseTerminal => {
261 let terminal_status = match (
265 candidate_class,
266 candidate.response_terminality,
267 ) {
268 (
269 PeerInputClass::ResponseTerminal,
270 Some(meerkat_core::interaction::TerminalityClass::Terminal {
271 disposition,
272 }),
273 ) => match disposition {
274 meerkat_core::interaction::TerminalDisposition::Completed => {
275 Some(meerkat_core::handles::PeerTerminalDisposition::Completed)
276 }
277 meerkat_core::interaction::TerminalDisposition::Failed => {
278 Some(meerkat_core::handles::PeerTerminalDisposition::Failed)
279 }
280 _ => {
281 reject_peer_response_observation_via_authority(
282 &comms_runtime,
283 &candidate,
284 "unsupported terminal disposition",
285 );
286 tracing::warn!(
287 class = ?candidate_class,
288 disposition = ?disposition,
289 interaction_id = %candidate.interaction.id,
290 "comms_drain: rejected peer response with unsupported terminal disposition"
291 );
292 continue;
293 }
294 },
295 (
296 PeerInputClass::ResponseProgress,
297 Some(meerkat_core::interaction::TerminalityClass::Progress),
298 ) => None,
299 (class, terminality) => {
300 reject_peer_response_observation_via_authority(
301 &comms_runtime,
302 &candidate,
303 "missing or inconsistent machine terminality",
304 );
305 tracing::warn!(
306 class = ?class,
307 terminality = ?terminality,
308 interaction_id = %candidate.interaction.id,
309 "comms_drain: rejected peer response with missing or inconsistent machine terminality"
310 );
311 continue;
312 }
313 };
314 let is_terminal = terminal_status.is_some();
315
316 if is_terminal {
317 let interaction_id = match &candidate.interaction.content {
324 meerkat_core::interaction::InteractionContent::Response {
325 in_reply_to,
326 ..
327 } => *in_reply_to,
328 other => {
329 tracing::warn!(
330 content = ?other,
331 "comms_drain: terminal response candidate missing response content"
332 );
333 continue;
334 }
335 };
336
337 let subscriber = comms_runtime.interaction_subscriber(&interaction_id);
349 let peer_interaction_handle = comms_runtime.peer_interaction_handle();
350 let dsl_installed = peer_interaction_handle.is_some();
351 if let (Some(handle), Some(disposition)) =
352 (peer_interaction_handle.as_ref(), terminal_status)
353 {
354 let corr_id =
355 meerkat_core::PeerCorrelationId::from_uuid(interaction_id.0);
356 if let Err(err) = handle.response_terminal(corr_id, disposition) {
357 tracing::warn!(
358 error = %err,
359 corr_id = %corr_id,
360 "PeerInteractionHandle::response_terminal rejected (no DSL entry — classified drain saw an unknown corr_id)"
361 );
362 reject_peer_response_observation_via_authority(
363 &comms_runtime,
364 &candidate,
365 "generated peer terminal transition rejected",
366 );
367 continue;
368 }
369 }
370
371 if comms_runtime.has_bridge_reply_waiter(&interaction_id) {
381 match comms_runtime.take_bridge_reply_waiter(&interaction_id) {
382 Some(waiter) => {
383 let _ = waiter.send(candidate.clone());
384 }
385 None => {
386 tracing::debug!(
387 interaction_id = %interaction_id,
388 "comms_drain: discarded late bridge reply for tombstoned waiter"
389 );
390 }
391 }
392 comms_runtime.mark_interaction_complete(&interaction_id);
393 continue;
394 }
395
396 let content_input = match classified_interaction_to_runtime_input(
397 &candidate,
398 &runtime_id,
399 ) {
400 Ok(input) => input,
401 Err(err) => {
402 tracing::warn!(
403 error = %err,
404 interaction_id = %interaction_id,
405 "comms_drain: rejected malformed terminal peer ingress"
406 );
407 if dsl_installed {
408 comms_runtime.abandon_interaction_stream(
409 &interaction_id,
410 meerkat_core::InteractionStreamAbandonReason::ResponseRejected,
411 );
412 } else {
413 comms_runtime.mark_interaction_complete(&interaction_id);
414 }
415 continue;
416 }
417 };
418 let result = adapter
419 .accept_input_with_completion(&session_id, content_input)
420 .await;
421 match result {
422 Ok((_outcome, handle)) => {
423 if subscriber.is_some() || handle.is_some() {
424 spawn_completion_bridge(
425 Some(comms_runtime.clone()),
426 interaction_id,
427 subscriber,
428 handle,
429 );
430 } else if !dsl_installed {
431 comms_runtime.mark_interaction_complete(&interaction_id);
432 }
433 }
434 Err(err) => {
435 tracing::warn!(
436 error = %err,
437 "comms_drain: failed to inject terminal response"
438 );
439 if dsl_installed {
440 comms_runtime.abandon_interaction_stream(
441 &interaction_id,
442 meerkat_core::InteractionStreamAbandonReason::ResponseRejected,
443 );
444 } else {
445 comms_runtime.mark_interaction_complete(&interaction_id);
446 }
447 }
448 }
449 } else {
450 if let Some(handle) = comms_runtime.peer_interaction_handle()
455 && let meerkat_core::interaction::InteractionContent::Response {
456 in_reply_to,
457 ..
458 } = &candidate.interaction.content
459 {
460 let corr_id =
461 meerkat_core::PeerCorrelationId::from_uuid(in_reply_to.0);
462 if let Err(err) = handle.response_progress(corr_id) {
463 tracing::warn!(
464 error = %err,
465 corr_id = %corr_id,
466 "PeerInteractionHandle::response_progress rejected"
467 );
468 reject_peer_response_observation_via_authority(
469 &comms_runtime,
470 &candidate,
471 "generated peer progress transition rejected",
472 );
473 continue;
474 }
475 }
476
477 if let meerkat_core::interaction::InteractionContent::Response {
485 in_reply_to,
486 ..
487 } = &candidate.interaction.content
488 && comms_runtime.has_bridge_reply_waiter(in_reply_to)
489 {
490 continue;
491 }
492
493 let input = match classified_interaction_to_runtime_input(
494 &candidate,
495 &runtime_id,
496 ) {
497 Ok(input) => input,
498 Err(err) => {
499 tracing::warn!(
500 error = %err,
501 interaction_id = %candidate.interaction.id,
502 "comms_drain: rejected malformed progress peer ingress"
503 );
504 continue;
505 }
506 };
507 if let Err(err) = adapter.accept_input(&session_id, input).await {
508 tracing::warn!(
509 error = %err,
510 "comms_drain: failed to inject progress response"
511 );
512 }
513 }
514 }
515 PeerInputClass::SilentRequest
516 | PeerInputClass::PeerLifecycleKickoffFailed
517 | PeerInputClass::PeerLifecycleKickoffCancelled
518 | PeerInputClass::ActionableMessage
519 | PeerInputClass::ActionableRequest
520 | PeerInputClass::PlainEvent => {
521 let interaction_id = candidate.interaction.id;
523
524 let fenced_member = if let meerkat_core::interaction::InteractionContent::IncarnationFencedMessage {
525 expected_recipient,
526 ..
527 } = &candidate.interaction.content
528 {
529 let expected_member = BridgeMemberIncarnation {
530 mob_id: expected_recipient.mob_id.clone(),
531 agent_identity: expected_recipient.agent_identity.clone(),
532 host_id: expected_recipient.host_id.clone(),
533 binding_generation: expected_recipient.binding_generation,
534 member_session_id: expected_recipient.member_session_id.clone(),
535 generation: expected_recipient.generation,
536 fence_token: expected_recipient.fence_token,
537 };
538 Some(expected_member)
539 } else {
540 None
541 };
542
543 let is_inbound_peer_request = matches!(
553 candidate_class,
554 PeerInputClass::SilentRequest | PeerInputClass::ActionableRequest
555 );
556 if is_inbound_peer_request {
557 let Some(handle) =
558 comms_runtime.peer_request_response_authority_handle()
559 else {
560 tracing::warn!(
561 interaction_id = %interaction_id,
562 class = ?candidate_class,
563 "comms_drain: rejected inbound peer request without complete peer request authority"
564 );
565 comms_runtime.abandon_interaction_stream(
566 &interaction_id,
567 meerkat_core::InteractionStreamAbandonReason::AdmissionRejected,
568 );
569 continue;
570 };
571 let corr_id =
572 meerkat_core::PeerCorrelationId::from_uuid(interaction_id.0);
573 if handle.inbound_state(corr_id).is_none()
574 && let Err(err) = handle
575 .request_received(corr_id, candidate.interaction.handling_mode)
576 {
577 tracing::warn!(
578 error = %err,
579 corr_id = %corr_id,
580 "PeerInteractionHandle::request_received rejected"
581 );
582 comms_runtime.abandon_interaction_stream(
583 &interaction_id,
584 meerkat_core::InteractionStreamAbandonReason::AdmissionRejected,
585 );
586 continue;
587 }
588 }
589
590 let subscriber = comms_runtime.interaction_subscriber(&interaction_id);
591 let input = match classified_interaction_to_runtime_input(
592 &candidate,
593 &runtime_id,
594 ) {
595 Ok(input) => input,
596 Err(err) => {
597 tracing::warn!(
598 error = %err,
599 interaction_id = %interaction_id,
600 "comms_drain: rejected malformed peer ingress"
601 );
602 comms_runtime.abandon_interaction_stream(
603 &interaction_id,
604 meerkat_core::InteractionStreamAbandonReason::AdmissionRejected,
605 );
606 continue;
607 }
608 };
609 let result = match fenced_member.as_ref() {
610 Some(expected_member) => {
611 adapter
612 .accept_input_with_completion_for_member_residency(
613 &session_id,
614 input,
615 Some(expected_member),
616 )
617 .await
618 }
619 None => {
620 adapter
621 .accept_input_with_completion(&session_id, input)
622 .await
623 }
624 };
625
626 match result {
627 Ok((_outcome, handle)) => {
628 if subscriber.is_some() || handle.is_some() {
629 spawn_completion_bridge(
630 Some(comms_runtime.clone()),
631 interaction_id,
632 subscriber,
633 handle,
634 );
635 } else {
636 comms_runtime.mark_interaction_complete(&interaction_id);
637 }
638 }
639 Err(err) => {
640 tracing::warn!(
641 error = %err,
642 "comms_drain: failed to accept peer input"
643 );
644 comms_runtime.abandon_interaction_stream(
645 &interaction_id,
646 meerkat_core::InteractionStreamAbandonReason::AdmissionRejected,
647 );
648 }
649 }
650 }
651 }
652 }
653 }
654 })
655}
656
657fn reject_peer_response_observation_via_authority(
658 comms_runtime: &Arc<dyn CommsRuntime>,
659 candidate: &PeerInputCandidate,
660 reason: &'static str,
661) {
662 let InteractionContent::Response { in_reply_to, .. } = &candidate.interaction.content else {
663 tracing::warn!(
664 reason = reason,
665 interaction_id = %candidate.interaction.id,
666 "comms_drain: cannot reject malformed peer response observation without response correlation"
667 );
668 return;
669 };
670 let corr_id = meerkat_core::PeerCorrelationId::from_uuid(in_reply_to.0);
671 let Some(handle) = comms_runtime.peer_interaction_handle() else {
672 tracing::warn!(
673 reason = reason,
674 corr_id = %corr_id,
675 interaction_id = %candidate.interaction.id,
676 "comms_drain: malformed peer response observation has no peer-interaction authority"
677 );
678 return;
679 };
680 if let Err(err) = handle.response_rejected(corr_id) {
681 tracing::warn!(
682 error = %err,
683 reason = reason,
684 corr_id = %corr_id,
685 interaction_id = %candidate.interaction.id,
686 "PeerInteractionHandle::response_rejected rejected malformed peer response observation"
687 );
688 return;
689 }
690 comms_runtime.abandon_interaction_stream(
691 in_reply_to,
692 meerkat_core::InteractionStreamAbandonReason::ResponseRejected,
693 );
694}
695
696fn bridge_peer_identity(
697 peer: &BridgePeerSpec,
698 context: &str,
699) -> Result<BridgePeerIdentity, (BridgeRejectionCause, String)> {
700 BridgePeerIdentity::try_from(peer).map_err(|error| {
701 (
702 BridgeRejectionCause::InvalidSupervisorSpec,
703 format!("{context}: invalid supervisor peer spec: {error}"),
704 )
705 })
706}
707
708fn sender_peer_label(sender: &PeerIngressFact) -> String {
709 sender.diagnostic_label()
710}
711
712fn sender_matches_bridge_peer(sender: &PeerIngressFact, peer: &BridgePeerIdentity) -> bool {
713 sender
714 .canonical_peer_id
715 .is_some_and(|sender_peer_id| sender_peer_id == peer.peer_id)
716 || (!peer.pubkey.is_zero() && sender.signing_pubkey == Some(*peer.pubkey.as_bytes()))
717}
718
719fn generated_sender_peer_id(sender: &PeerIngressFact) -> Option<String> {
720 sender.canonical_peer_id.map(|peer_id| peer_id.as_str())
721}
722
723fn generated_sender_signing_public_key(sender: &PeerIngressFact) -> Option<String> {
724 sender
725 .signing_pubkey
726 .map(encode_supervisor_signing_public_key)
727}
728
729fn supervisor_bind_admission_rejection_cause(
730 rejection: crate::meerkat_machine::dsl::SupervisorBindRejectionKind,
731) -> BridgeRejectionCause {
732 match rejection {
733 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::AlreadyBound => {
734 BridgeRejectionCause::AlreadyBound
735 }
736 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::SenderMismatch => {
737 BridgeRejectionCause::SenderMismatch
738 }
739 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::RevocationPending => {
740 BridgeRejectionCause::AlreadyBound
741 }
742 }
743}
744
745fn supervisor_bind_admission_rejection_reason(
746 rejection: crate::meerkat_machine::dsl::SupervisorBindRejectionKind,
747 sender: &PeerIngressFact,
748 supervisor: &BridgePeerIdentity,
749) -> String {
750 match rejection {
751 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::AlreadyBound => {
752 "bind member admission rejected by MeerkatMachine: supervisor already bound; use authorize_supervisor to rotate"
753 .to_string()
754 }
755 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::SenderMismatch => {
756 let sender_label = sender_peer_label(sender);
757 format!(
758 "bind member admission rejected by MeerkatMachine: request sender '{sender_label}' does not match authorized supervisor '{}'",
759 supervisor.peer_id
760 )
761 }
762 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::RevocationPending => {
763 "bind member admission rejected by MeerkatMachine: prior supervisor revocation is still pending; retry revoke_supervisor before binding"
764 .to_string()
765 }
766 }
767}
768
769fn supervisor_authorize_admission_rejection_cause(
770 rejection: crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind,
771) -> BridgeRejectionCause {
772 match rejection {
773 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::NotBound => {
774 BridgeRejectionCause::NotBound
775 }
776 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::StaleSupervisor => {
777 BridgeRejectionCause::StaleSupervisor
778 }
779 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::SenderMismatch => {
780 BridgeRejectionCause::SenderMismatch
781 }
782 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::RotationNotAllowed => {
783 BridgeRejectionCause::Unsupported
784 }
785 }
786}
787
788fn supervisor_authorize_admission_rejection_reason(
789 rejection: crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind,
790 sender: &PeerIngressFact,
791 supervisor: &BridgePeerIdentity,
792) -> String {
793 match rejection {
794 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::NotBound => {
795 "authorize supervisor admission rejected by MeerkatMachine: use bind_member to establish initial supervisor authority"
796 .to_string()
797 }
798 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::StaleSupervisor => {
799 "authorize supervisor admission rejected by MeerkatMachine: stale supervisor binding"
800 .to_string()
801 }
802 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::SenderMismatch => {
803 let sender_label = sender_peer_label(sender);
804 format!(
805 "authorize supervisor admission rejected by MeerkatMachine: request sender '{sender_label}' does not match authorized supervisor '{}'",
806 supervisor.peer_id
807 )
808 }
809 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::RotationNotAllowed => {
810 "authorize supervisor admission rejected by MeerkatMachine: supervisor rotation is not allowed after the member runtime has stopped accepting ordinary commands"
811 .to_string()
812 }
813 }
814}
815
816fn supervisor_bridge_admission_rejection_cause(
817 rejection: crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind,
818) -> BridgeRejectionCause {
819 match rejection {
820 crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind::NotBound => {
821 BridgeRejectionCause::NotBound
822 }
823 crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind::StaleSupervisor => {
824 BridgeRejectionCause::StaleSupervisor
825 }
826 crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind::SenderMismatch => {
827 BridgeRejectionCause::SenderMismatch
828 }
829 crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind::CommandNotAllowed => {
830 BridgeRejectionCause::Unsupported
831 }
832 }
833}
834
835fn supervisor_bridge_admission_rejection_reason(
836 rejection: crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind,
837 sender: &PeerIngressFact,
838 supervisor: &BridgePeerIdentity,
839) -> String {
840 match rejection {
841 crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind::NotBound => {
842 "supervisor bridge admission rejected by MeerkatMachine: no authorized supervisor registered"
843 .to_string()
844 }
845 crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind::StaleSupervisor => {
846 "supervisor bridge admission rejected by MeerkatMachine: stale supervisor binding"
847 .to_string()
848 }
849 crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind::SenderMismatch => {
850 let sender_label = sender_peer_label(sender);
851 format!(
852 "supervisor bridge admission rejected by MeerkatMachine: request sender '{sender_label}' does not match authorized supervisor '{}'",
853 supervisor.peer_id
854 )
855 }
856 crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind::CommandNotAllowed => {
857 "supervisor bridge admission rejected by MeerkatMachine: command is not allowed in the current runtime phase"
858 .to_string()
859 }
860 }
861}
862
863struct BridgeMobPeerOverlay {
864 endpoints: BTreeSet<crate::meerkat_machine::dsl::PeerEndpoint>,
865 endpoint_count: u64,
866}
867
868fn bridge_mob_peer_overlay(
869 handoff: &BridgeMobPeerOverlayHandoff,
870) -> Result<BridgeMobPeerOverlay, String> {
871 let mut endpoints = BTreeSet::new();
872 let endpoint_count = u64::try_from(handoff.peer_specs.len())
873 .map_err(|_| "MobMachine peer overlay contains too many endpoints".to_string())?;
874 for peer_spec in &handoff.peer_specs {
875 let peer = TrustedPeerDescriptor::try_from(peer_spec)?;
876 let endpoint = crate::meerkat_machine::dsl::PeerEndpoint::from(&peer);
877 endpoints.insert(endpoint);
878 }
879 Ok(BridgeMobPeerOverlay {
880 endpoints,
881 endpoint_count,
882 })
883}
884
885async fn resolve_authorized_supervisor(
890 adapter: &MeerkatMachine,
891 session_id: &SessionId,
892 sender: &PeerIngressFact,
893 payload: &BridgeSupervisorPayload,
894) -> Result<BridgePeerIdentity, (BridgeRejectionCause, String)> {
895 let supervisor = bridge_peer_identity(&payload.supervisor, "supervisor bridge request")?;
896 let sender_peer_id = sender
897 .canonical_peer_id
898 .map(|sender_peer_id| sender_peer_id.to_string());
899 match adapter
900 .resolve_supervisor_bridge_command_admission(
901 session_id,
902 supervisor.peer_id.to_string(),
903 payload.epoch,
904 sender_peer_id,
905 )
906 .await
907 {
908 Ok(crate::meerkat_machine::SupervisorBridgeCommandAdmission::Accepted) => Ok(supervisor),
909 Ok(crate::meerkat_machine::SupervisorBridgeCommandAdmission::ResumePendingRevoke) => Err((
910 BridgeRejectionCause::Internal,
911 "ordinary supervisor admission returned a cleanup-only pending-revoke disposition"
912 .to_string(),
913 )),
914 Ok(crate::meerkat_machine::SupervisorBridgeCommandAdmission::Rejected(rejection)) => Err((
915 supervisor_bridge_admission_rejection_cause(rejection),
916 supervisor_bridge_admission_rejection_reason(rejection, sender, &supervisor),
917 )),
918 Err(error) => Err((
919 BridgeRejectionCause::Internal,
920 format!("supervisor bridge admission failed: {error}"),
921 )),
922 }
923}
924
925fn supervisor_cleanup_command_label(
926 command_kind: crate::meerkat_machine::dsl::SupervisorCleanupCommandKind,
927) -> &'static str {
928 match command_kind {
929 crate::meerkat_machine::dsl::SupervisorCleanupCommandKind::Retire => "retire_member",
930 crate::meerkat_machine::dsl::SupervisorCleanupCommandKind::Observe => "observe_member",
931 crate::meerkat_machine::dsl::SupervisorCleanupCommandKind::Destroy => "destroy_member",
932 crate::meerkat_machine::dsl::SupervisorCleanupCommandKind::Revoke => "revoke_supervisor",
933 }
934}
935
936struct AuthorizedSupervisorCleanup {
940 supervisor: BridgePeerIdentity,
941 resume_pending_revoke: bool,
942}
943
944async fn resolve_authorized_supervisor_cleanup(
945 adapter: &MeerkatMachine,
946 session_id: &SessionId,
947 sender: &PeerIngressFact,
948 payload: &BridgeSupervisorPayload,
949 command_kind: crate::meerkat_machine::dsl::SupervisorCleanupCommandKind,
950) -> Result<AuthorizedSupervisorCleanup, (BridgeRejectionCause, String)> {
951 let supervisor = bridge_peer_identity(&payload.supervisor, "supervisor cleanup request")?;
952 let sender_peer_id = sender
953 .canonical_peer_id
954 .map(|sender_peer_id| sender_peer_id.to_string());
955 match adapter
956 .resolve_supervisor_cleanup_command_admission(
957 session_id,
958 command_kind,
959 supervisor.peer_id.to_string(),
960 payload.epoch,
961 sender_peer_id,
962 )
963 .await
964 {
965 Ok((crate::meerkat_machine::SupervisorBridgeCommandAdmission::Accepted, _)) => {
966 Ok(AuthorizedSupervisorCleanup {
967 supervisor,
968 resume_pending_revoke: false,
969 })
970 }
971 Ok((crate::meerkat_machine::SupervisorBridgeCommandAdmission::ResumePendingRevoke, _)) => {
972 Ok(AuthorizedSupervisorCleanup {
973 supervisor,
974 resume_pending_revoke: true,
975 })
976 }
977 Ok((
978 crate::meerkat_machine::SupervisorBridgeCommandAdmission::Rejected(rejection),
979 phase,
980 )) => {
981 let reason = if rejection
982 == crate::meerkat_machine::dsl::SupervisorBridgeCommandRejectionKind::CommandNotAllowed
983 {
984 format!(
985 "supervisor cleanup command '{}' is not allowed while the member runtime is {phase:?}",
986 supervisor_cleanup_command_label(command_kind)
987 )
988 } else {
989 supervisor_bridge_admission_rejection_reason(rejection, sender, &supervisor)
990 };
991 Err((
992 supervisor_bridge_admission_rejection_cause(rejection),
993 reason,
994 ))
995 }
996 Err(error) => Err((
997 BridgeRejectionCause::Internal,
998 format!("supervisor cleanup admission failed: {error}"),
999 )),
1000 }
1001}
1002
1003async fn resolve_authorized_supervisor_with_response_route(
1008 adapter: &Arc<MeerkatMachine>,
1009 session_id: &SessionId,
1010 comms_runtime: &Arc<dyn CommsRuntime>,
1011 sender: &PeerIngressFact,
1012 payload: &BridgeSupervisorPayload,
1013 context: &str,
1014) -> Result<BridgePeerIdentity, (BridgeRejectionCause, String)> {
1015 let supervisor = resolve_authorized_supervisor(adapter, session_id, sender, payload).await?;
1016 let response_peer = supervisor.clone().into_trusted_peer_descriptor();
1017 refresh_and_publish_bound_supervisor_route(
1018 adapter,
1019 session_id,
1020 comms_runtime.as_ref(),
1021 &response_peer,
1022 payload.epoch,
1023 generated_sender_peer_id(sender),
1024 context,
1025 )
1026 .await
1027 .map_err(|reason| (BridgeRejectionCause::Internal, reason))?;
1028 Ok(supervisor)
1029}
1030
1031async fn resolve_authorized_supervisor_cleanup_with_response_route(
1032 adapter: &Arc<MeerkatMachine>,
1033 session_id: &SessionId,
1034 comms_runtime: &Arc<dyn CommsRuntime>,
1035 sender: &PeerIngressFact,
1036 payload: &BridgeSupervisorPayload,
1037 command_kind: crate::meerkat_machine::dsl::SupervisorCleanupCommandKind,
1038 context: &str,
1039) -> Result<BridgePeerIdentity, (BridgeRejectionCause, String)> {
1040 let supervisor =
1041 resolve_authorized_supervisor_cleanup(adapter, session_id, sender, payload, command_kind)
1042 .await?;
1043 if supervisor.resume_pending_revoke {
1044 return Err((
1045 BridgeRejectionCause::Internal,
1046 format!("{context}: pending revoke disposition is invalid for this command"),
1047 ));
1048 }
1049 let response_peer = supervisor.supervisor.clone().into_trusted_peer_descriptor();
1050 refresh_and_publish_bound_supervisor_route(
1051 adapter,
1052 session_id,
1053 comms_runtime.as_ref(),
1054 &response_peer,
1055 payload.epoch,
1056 generated_sender_peer_id(sender),
1057 context,
1058 )
1059 .await
1060 .map_err(|reason| (BridgeRejectionCause::Internal, reason))?;
1061 Ok(supervisor.supervisor)
1062}
1063
1064fn bridge_capabilities(
1065 origin_protocol: meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion,
1066) -> BridgeCapabilities {
1067 let mut capabilities = BridgeCapabilities {
1068 deliver_member_input: true,
1069 observe_member: true,
1070 interrupt_member: true,
1071 hard_cancel_member: origin_protocol.supports_multi_host(),
1074 tracked_input_cancel: origin_protocol.supports_multi_host(),
1075 retire_member: true,
1076 destroy_member: true,
1077 wire_member: true,
1078 unwire_member: true,
1079 ..BridgeCapabilities::default()
1080 };
1081 if !origin_protocol.supports_multi_host() {
1082 capabilities.current_protocol_version = origin_protocol;
1083 capabilities.default_protocol_version = origin_protocol;
1084 capabilities.supported_protocol_versions = match origin_protocol {
1085 version
1086 if version
1087 == meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion::V2 =>
1088 {
1089 vec![meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion::V2]
1090 }
1091 _ => vec![
1092 meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion::V2,
1093 meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion::V3,
1094 ],
1095 };
1096 }
1097 capabilities
1098}
1099
1100fn peer_input_from_delivery_payload(
1101 session_id: &SessionId,
1102 sender_peer_id: PeerId,
1103 payload: BridgeDeliveryPayload,
1104) -> Result<Input, String> {
1105 if payload.turn.is_some() {
1106 return Err(
1107 "directive-bearing member delivery must use the tracked flow-step lowering path"
1108 .to_string(),
1109 );
1110 }
1111 let stable_uuid = canonical_non_nil_delivery_uuid("input_id", &payload.input_id)?;
1112 let transcript_uuid = payload
1113 .transcript_interaction_id
1114 .as_deref()
1115 .map(|raw| canonical_non_nil_delivery_uuid("transcript_interaction_id", raw))
1116 .transpose()?;
1117 let is_placed = payload.expected_member.is_some();
1118 let directed_interaction_id = delivery_requests_tracked_interaction(&payload)
1119 .then_some(meerkat_core::interaction::InteractionId(stable_uuid));
1120 let correlation_id = transcript_uuid
1121 .or_else(|| (!is_placed).then_some(stable_uuid))
1122 .map(crate::identifiers::CorrelationId::from_uuid);
1123 let header = InputHeader {
1124 id: meerkat_core::lifecycle::InputId::from_uuid(stable_uuid),
1125 timestamp: chrono::Utc::now(),
1126 source: InputOrigin::Peer {
1127 peer_id: sender_peer_id.as_str(),
1128 display_identity: Some(sender_peer_id.as_str()),
1129 runtime_id: Some(MeerkatMachine::logical_runtime_id(session_id)),
1130 },
1131 durability: InputDurability::Durable,
1132 visibility: InputVisibility {
1133 transcript_eligible: true,
1134 operator_eligible: true,
1135 },
1136 idempotency_key: Some(IdempotencyKey::new(payload.input_id.clone())),
1137 supersession_key: None,
1138 correlation_id,
1142 };
1143 if let Some(context) = payload.transient_turn_context {
1144 if !payload.injected_context.is_empty() {
1145 return Err(
1146 "transient_turn_context cannot be combined with injected_context on a plain bridge delivery"
1147 .to_string(),
1148 );
1149 }
1150 let mut turn_metadata =
1151 crate::runtime_loop::for_bridge_turn_directive(payload.handling_mode, None);
1152 turn_metadata.system_prompts = payload.system_prompt.into_iter().collect();
1153 turn_metadata.transient_turn_context = Some(context);
1154 turn_metadata.transcript_identity.objective_id = payload.objective_id;
1155 return Ok(Input::FlowStep(FlowStepInput {
1156 header,
1157 step_id: format!("bridge-delivery:{}", payload.input_id),
1158 content: payload.content,
1159 directed_interaction_id,
1160 turn_metadata: Some(turn_metadata),
1161 }));
1162 }
1163 Ok(Input::Peer(PeerInput {
1164 directed_interaction_id,
1165 objective_id: payload.objective_id,
1166 system_prompts: payload.system_prompt.into_iter().collect(),
1167 header,
1168 convention: Some(PeerConvention::Message),
1169 content: payload.content,
1170 payload: None,
1171 handling_mode: Some(payload.handling_mode),
1172 sender_taint: None,
1175 injected_context: payload.injected_context,
1179 }))
1180}
1181
1182fn delivery_requests_tracked_interaction(payload: &BridgeDeliveryPayload) -> bool {
1187 match payload.outcome_tracking {
1188 None => false,
1189 Some(BridgeOutcomeTracking::Interaction) => true,
1190 }
1191}
1192
1193fn delivery_requires_tracked_turn_custody(payload: &BridgeDeliveryPayload) -> bool {
1194 payload.turn.is_some() || delivery_requests_tracked_interaction(payload)
1195}
1196
1197fn validate_delivery_tracking_request(payload: &BridgeDeliveryPayload) -> Result<(), String> {
1198 canonical_non_nil_delivery_uuid("input_id", &payload.input_id)?;
1199 if payload.turn.is_some() && payload.outcome_tracking.is_some() {
1200 return Err(
1201 "turn and outcome_tracking are mutually exclusive terminal-custody requests"
1202 .to_string(),
1203 );
1204 }
1205 if payload.turn.is_some() && payload.system_prompt.is_some() {
1206 return Err("system_prompt is not supported on directive-bearing deliveries".to_string());
1207 }
1208 if payload.outcome_tracking.is_some() && !payload.protocol_version.supports_multi_host() {
1209 return Err(
1210 "outcome_tracking requires a V4 supervisor bridge protocol version".to_string(),
1211 );
1212 }
1213 if payload.outcome_tracking.is_some() && payload.expected_member.is_none() {
1214 return Err("outcome_tracking requires an exact placed-member incarnation".to_string());
1215 }
1216 if let Some(transcript_id) = payload.transcript_interaction_id.as_deref() {
1217 if !payload.protocol_version.supports_multi_host() {
1218 return Err(
1219 "transcript_interaction_id requires a V4 supervisor bridge protocol version"
1220 .to_string(),
1221 );
1222 }
1223 if payload.expected_member.is_none() {
1224 return Err(
1225 "transcript_interaction_id requires an exact placed-member incarnation".to_string(),
1226 );
1227 }
1228 if payload.turn.is_some() {
1229 return Err(
1230 "turn and transcript_interaction_id are mutually exclusive identity carriers"
1231 .to_string(),
1232 );
1233 }
1234 canonical_non_nil_delivery_uuid("transcript_interaction_id", transcript_id)?;
1235 }
1236 if payload.outcome_tracking == Some(BridgeOutcomeTracking::Interaction)
1237 && payload.transcript_interaction_id.as_deref() != Some(payload.input_id.as_str())
1238 {
1239 return Err(
1240 "tracked interaction requires transcript_interaction_id to equal input_id".to_string(),
1241 );
1242 }
1243 Ok(())
1244}
1245
1246fn canonical_non_nil_delivery_uuid(field: &str, raw: &str) -> Result<uuid::Uuid, String> {
1247 let parsed = uuid::Uuid::parse_str(raw)
1248 .map_err(|error| format!("delivery {field} '{raw}' is not a UUID: {error}"))?;
1249 if parsed.is_nil() || parsed.to_string() != raw {
1250 return Err(format!(
1251 "delivery {field} '{raw}' must be a canonical non-nil UUID"
1252 ));
1253 }
1254 Ok(parsed)
1255}
1256
1257fn bridge_delivery_rejection_cause(
1258 reason: &crate::accept::RejectReason,
1259) -> BridgeDeliveryRejectionCause {
1260 match reason {
1261 crate::accept::RejectReason::NotReady { state } => BridgeDeliveryRejectionCause::NotReady {
1262 state: runtime_state_to_bridge(*state),
1263 },
1264 crate::accept::RejectReason::DurabilityViolation { detail } => {
1265 BridgeDeliveryRejectionCause::DurabilityViolation {
1266 detail: detail.clone(),
1267 }
1268 }
1269 crate::accept::RejectReason::PeerHandlingModeInvalid { detail } => {
1270 BridgeDeliveryRejectionCause::PeerHandlingModeInvalid {
1271 detail: detail.clone(),
1272 }
1273 }
1274 crate::accept::RejectReason::PeerResponseTerminalInvalid { detail } => {
1275 BridgeDeliveryRejectionCause::Internal {
1276 detail: detail.clone(),
1277 }
1278 }
1279 }
1280}
1281
1282fn advertised_bind_bootstrap_token(
1283 comms_runtime: &Arc<dyn CommsRuntime>,
1284) -> Result<String, (BridgeRejectionCause, String)> {
1285 if let Some(token) = comms_runtime.bridge_bootstrap_token()
1286 && !token.is_empty()
1287 {
1288 return Ok(token);
1289 }
1290 Err((
1291 BridgeRejectionCause::InvalidBootstrapToken,
1292 "runtime does not expose a typed bridge bootstrap token".to_string(),
1293 ))
1294}
1295
1296async fn validate_bind_request(
1312 adapter: &Arc<MeerkatMachine>,
1313 session_id: &SessionId,
1314 comms_runtime: &Arc<dyn CommsRuntime>,
1315 sender: &PeerIngressFact,
1316 payload: &meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload,
1317) -> Result<(TrustedPeerDescriptor, String), (BridgeRejectionCause, String)> {
1318 let expected_bootstrap_token = advertised_bind_bootstrap_token(comms_runtime)?;
1319 let advertised_address = comms_runtime.advertised_address().ok_or_else(|| {
1320 (
1321 BridgeRejectionCause::Internal,
1322 "runtime does not expose an advertised address for bind bootstrap".to_string(),
1323 )
1324 })?;
1325 let supervisor = bridge_peer_identity(&payload.supervisor, "bind member failed")?;
1326 let Some(actual_peer_id) = comms_runtime.peer_id().map(|peer_id| peer_id.as_str()) else {
1331 return Err((
1332 BridgeRejectionCause::Internal,
1333 "bind member failed: runtime peer_id unavailable".to_string(),
1334 ));
1335 };
1336
1337 let address_matches = canonicalize_bridge_address(&payload.expected_address)
1340 == canonicalize_bridge_address(&advertised_address);
1341 let sender_matches_supervisor = sender_matches_bridge_peer(sender, &supervisor);
1342 let expected_peer_id_matches = actual_peer_id == payload.expected_peer_id;
1343 let bootstrap_token_matches = payload.bootstrap_token.as_str() == expected_bootstrap_token;
1344
1345 let verdict = adapter
1346 .resolve_supervisor_bind_material_admission(
1347 session_id,
1348 address_matches,
1349 sender_matches_supervisor,
1350 expected_peer_id_matches,
1351 bootstrap_token_matches,
1352 )
1353 .await
1354 .map_err(|error| {
1355 (
1356 BridgeRejectionCause::Internal,
1357 format!("bind member material admission failed: {error}"),
1358 )
1359 })?;
1360
1361 match verdict {
1362 crate::meerkat_machine::dsl::SupervisorBindMaterialAdmissionKind::Accept => Ok((
1363 supervisor.into_trusted_peer_descriptor(),
1364 advertised_address,
1365 )),
1366 crate::meerkat_machine::dsl::SupervisorBindMaterialAdmissionKind::AddressMismatch => Err((
1367 BridgeRejectionCause::AddressMismatch,
1368 format!(
1369 "bind address mismatch: expected '{}', actual '{}'",
1370 payload.expected_address, advertised_address
1371 ),
1372 )),
1373 crate::meerkat_machine::dsl::SupervisorBindMaterialAdmissionKind::SenderMismatch => {
1374 let sender_label = sender_peer_label(sender);
1375 Err((
1376 BridgeRejectionCause::SenderMismatch,
1377 format!(
1378 "request sender '{sender_label}' does not match supervisor '{}'",
1379 supervisor.peer_id
1380 ),
1381 ))
1382 }
1383 crate::meerkat_machine::dsl::SupervisorBindMaterialAdmissionKind::InvalidPeerSpec => Err((
1384 BridgeRejectionCause::InvalidPeerSpec,
1385 format!(
1386 "bind peer_id mismatch: expected '{}', actual '{actual_peer_id}'",
1387 payload.expected_peer_id
1388 ),
1389 )),
1390 crate::meerkat_machine::dsl::SupervisorBindMaterialAdmissionKind::InvalidBootstrapToken => {
1391 Err((
1392 BridgeRejectionCause::InvalidBootstrapToken,
1393 "bind member failed: invalid bootstrap token".to_string(),
1394 ))
1395 }
1396 }
1397}
1398
1399#[derive(Clone, Debug)]
1400struct BoundSupervisorState {
1401 name: String,
1402 peer_id: String,
1403 address: String,
1404 signing_public_key: String,
1405 epoch: u64,
1406}
1407
1408impl BoundSupervisorState {
1409 fn from_binding(binding: &SupervisorBinding) -> Option<Self> {
1412 let SupervisorBinding::Bound {
1413 name,
1414 peer_id,
1415 address,
1416 signing_public_key,
1417 epoch,
1418 } = binding
1419 else {
1420 return None;
1421 };
1422 Some(Self {
1423 name: name.clone(),
1424 peer_id: peer_id.clone(),
1425 address: address.clone(),
1426 signing_public_key: signing_public_key.clone(),
1427 epoch: *epoch,
1428 })
1429 }
1430}
1431
1432pub fn encode_supervisor_signing_public_key(pubkey: [u8; 32]) -> String {
1433 const HEX: &[u8; 16] = b"0123456789abcdef";
1434 let mut encoded = String::with_capacity(64);
1435 for byte in pubkey {
1436 encoded.push(HEX[(byte >> 4) as usize] as char);
1437 encoded.push(HEX[(byte & 0x0f) as usize] as char);
1438 }
1439 encoded
1440}
1441
1442pub fn decode_supervisor_signing_public_key(encoded: &str) -> Result<[u8; 32], String> {
1443 fn hex_nibble(byte: u8) -> Option<u8> {
1444 match byte {
1445 b'0'..=b'9' => Some(byte - b'0'),
1446 b'a'..=b'f' => Some(byte - b'a' + 10),
1447 b'A'..=b'F' => Some(byte - b'A' + 10),
1448 _ => None,
1449 }
1450 }
1451
1452 let bytes = encoded.as_bytes();
1453 if bytes.len() != 64 {
1454 return Err(format!(
1455 "supervisor signing public key must be 64 hex characters, got {}",
1456 bytes.len()
1457 ));
1458 }
1459 let mut decoded = [0u8; 32];
1460 for (index, chunk) in bytes.chunks_exact(2).enumerate() {
1461 let high = hex_nibble(chunk[0]).ok_or_else(|| {
1462 format!(
1463 "invalid supervisor signing public key hex at byte {}",
1464 index * 2
1465 )
1466 })?;
1467 let low = hex_nibble(chunk[1]).ok_or_else(|| {
1468 format!(
1469 "invalid supervisor signing public key hex at byte {}",
1470 index * 2 + 1
1471 )
1472 })?;
1473 decoded[index] = (high << 4) | low;
1474 }
1475 Ok(decoded)
1476}
1477
1478pub fn trusted_peer_descriptor_from_supervisor_publish_obligation(
1479 obligation: &crate::protocol_supervisor_trust_publish::SupervisorTrustPublishObligation,
1480) -> Result<TrustedPeerDescriptor, String> {
1481 let signing_public_key = obligation.signing_public_key().as_ref().ok_or_else(|| {
1482 "generated supervisor trust publish obligation omitted signing public key".to_string()
1483 })?;
1484 let pubkey = decode_supervisor_signing_public_key(signing_public_key)?;
1485 TrustedPeerDescriptor::unsigned_with_pubkey(
1486 obligation.name().clone(),
1487 obligation.peer_id().clone(),
1488 pubkey,
1489 obligation.address().clone(),
1490 )
1491}
1492
1493async fn single_supervisor_publish_obligation(
1494 adapter: &MeerkatMachine,
1495 session_id: &SessionId,
1496 transition: &crate::meerkat_machine::dsl::MeerkatMachineTransition,
1497 context: &str,
1498) -> Result<crate::protocol_supervisor_trust_publish::SupervisorTrustPublishObligation, String> {
1499 let freshness_authority = adapter
1500 .supervisor_trust_publish_freshness_authority(session_id)
1501 .await
1502 .map_err(|error| {
1503 format!("{context}: generated supervisor publish freshness unavailable: {error}")
1504 })?;
1505 let obligations = crate::protocol_supervisor_trust_publish::extract_obligations_with_freshness(
1506 transition,
1507 freshness_authority,
1508 );
1509 match obligations.as_slice() {
1510 [obligation] => Ok(obligation.clone()),
1511 [] => Err(format!("{context}: generated publish effect was absent")),
1512 _ => Err(format!(
1513 "{context}: generated multiple supervisor trust publish effects"
1514 )),
1515 }
1516}
1517
1518async fn matching_supervisor_revoke_obligation(
1519 adapter: &Arc<MeerkatMachine>,
1520 session_id: &SessionId,
1521 transition: &crate::meerkat_machine::dsl::MeerkatMachineTransition,
1522 peer_id: &str,
1523 epoch: u64,
1524 context: &str,
1525) -> Result<crate::protocol_supervisor_trust_revoke::SupervisorTrustRevokeObligation, String> {
1526 let freshness_authority = adapter
1527 .supervisor_trust_revoke_freshness_authority(session_id)
1528 .await
1529 .map_err(|error| {
1530 format!("{context}: generated supervisor revoke freshness unavailable: {error}")
1531 })?;
1532 crate::protocol_supervisor_trust_revoke::extract_obligations_with_freshness(
1533 transition,
1534 freshness_authority,
1535 )
1536 .into_iter()
1537 .find(|obligation| obligation.peer_id().as_str() == peer_id && obligation.epoch() == epoch)
1538 .ok_or_else(|| format!("{context}: generated revoke effect was absent"))
1539}
1540
1541fn validate_supervisor_publish_obligation(
1542 obligation: &crate::protocol_supervisor_trust_publish::SupervisorTrustPublishObligation,
1543 expected: &TrustedPeerDescriptor,
1544 expected_epoch: u64,
1545 context: &str,
1546) -> Result<(), String> {
1547 let expected_signing_key = encode_supervisor_signing_public_key(expected.pubkey);
1548 if obligation.name() != expected.name.as_str()
1549 || obligation.peer_id().as_str() != expected.peer_id.as_str()
1550 || obligation.address().as_str() != expected.address.to_string()
1551 || obligation.signing_public_key().as_deref() != Some(expected_signing_key.as_str())
1552 || obligation.epoch() != expected_epoch
1553 {
1554 return Err(format!(
1555 "{context}: generated publish obligation did not match the staged supervisor binding"
1556 ));
1557 }
1558 Ok(())
1559}
1560
1561async fn apply_generated_private_trust_add(
1562 comms_runtime: &dyn CommsRuntime,
1563 peer: TrustedPeerDescriptor,
1564 authority: CommsTrustMutationAuthority,
1565) -> Result<(), String> {
1566 match comms_runtime
1567 .apply_trust_mutation(CommsTrustMutation::AddPrivateTrustedPeer { peer, authority })
1568 .await
1569 .map_err(|error| error.to_string())?
1570 {
1571 CommsTrustMutationResult::Added { .. } => Ok(()),
1572 other => Err(format!(
1573 "generated trust add returned unexpected result: {other:?}"
1574 )),
1575 }
1576}
1577
1578async fn apply_generated_private_trust_remove(
1579 comms_runtime: &dyn CommsRuntime,
1580 peer_id: String,
1581 authority: CommsTrustMutationAuthority,
1582) -> Result<bool, String> {
1583 match comms_runtime
1584 .apply_trust_mutation(CommsTrustMutation::RemovePrivateTrustedPeer { peer_id, authority })
1585 .await
1586 .map_err(|error| error.to_string())?
1587 {
1588 CommsTrustMutationResult::Removed { removed } => Ok(removed),
1589 other => Err(format!(
1590 "generated trust remove returned unexpected result: {other:?}"
1591 )),
1592 }
1593}
1594
1595fn supervisor_publish_authority(
1596 obligation: &crate::protocol_supervisor_trust_publish::SupervisorTrustPublishObligation,
1597) -> Result<CommsTrustMutationAuthority, String> {
1598 crate::protocol_supervisor_trust_publish::publish_authority_for_peer(
1599 obligation,
1600 obligation.peer_id(),
1601 )
1602}
1603
1604fn supervisor_publish_cleanup_authority(
1605 obligation: &crate::protocol_supervisor_trust_publish::SupervisorTrustPublishObligation,
1606) -> Result<CommsTrustMutationAuthority, String> {
1607 crate::protocol_supervisor_trust_publish::cleanup_authority_for_peer(
1608 obligation,
1609 obligation.peer_id(),
1610 )
1611}
1612
1613fn supervisor_revoke_authority(
1614 obligation: &crate::protocol_supervisor_trust_revoke::SupervisorTrustRevokeObligation,
1615) -> Result<CommsTrustMutationAuthority, String> {
1616 crate::protocol_supervisor_trust_revoke::revoke_authority_for_peer(
1617 obligation,
1618 obligation.peer_id(),
1619 )
1620}
1621
1622async fn publish_supervisor_trust_from_generated_obligation(
1623 adapter: &MeerkatMachine,
1624 session_id: &SessionId,
1625 comms_runtime: &dyn CommsRuntime,
1626 obligation: &crate::protocol_supervisor_trust_publish::SupervisorTrustPublishObligation,
1627) -> Result<(), String> {
1628 let trusted_peer = trusted_peer_descriptor_from_supervisor_publish_obligation(obligation)?;
1629 let failure_cleanup_obligation = supervisor_route_publish_obligation(
1639 adapter,
1640 session_id,
1641 &trusted_peer,
1642 "supervisor trust route replacement failure cleanup",
1643 )
1644 .await?;
1645 let replacement_cleanup_authority = supervisor_publish_cleanup_authority(obligation)?;
1646 let failure_cleanup_authority =
1647 supervisor_publish_cleanup_authority(&failure_cleanup_obligation)?;
1648 apply_generated_private_trust_remove(
1649 comms_runtime,
1650 obligation.peer_id().clone(),
1651 replacement_cleanup_authority,
1652 )
1653 .await?;
1654 apply_generated_private_trust_add(
1655 comms_runtime,
1656 trusted_peer,
1657 supervisor_publish_authority(obligation)?,
1658 )
1659 .await?;
1660 if let Err(error) = adapter
1661 .stage_supervisor_trust_published(
1662 session_id,
1663 obligation.peer_id().clone(),
1664 obligation.epoch(),
1665 )
1666 .await
1667 {
1668 let cleanup_result = apply_generated_private_trust_remove(
1669 comms_runtime,
1670 obligation.peer_id().clone(),
1671 failure_cleanup_authority,
1672 )
1673 .await;
1674 let mut reason = format!("generated trust publish ack rejected: {error}");
1675 if let Err(cleanup_error) = cleanup_result {
1676 reason.push_str(&format!("; cleanup remove failed: {cleanup_error}"));
1677 }
1678 return Err(reason);
1679 }
1680 Ok(())
1681}
1682
1683async fn refresh_and_publish_bound_supervisor_route(
1684 adapter: &MeerkatMachine,
1685 session_id: &SessionId,
1686 comms_runtime: &dyn CommsRuntime,
1687 descriptor: &TrustedPeerDescriptor,
1688 epoch: u64,
1689 sender_peer_id: Option<String>,
1690 context: &str,
1691) -> Result<(), String> {
1692 adapter
1693 .stage_local_endpoint_for_comms_runtime(session_id, comms_runtime)
1694 .await
1695 .map_err(|error| format!("{context}: local endpoint rejected: {error}"))?;
1696 let transition = adapter
1697 .stage_supervisor_binding_route_refresh(
1698 session_id,
1699 GeneratedSupervisorBinding {
1700 name: descriptor.name.as_str().to_owned(),
1701 peer_id: descriptor.peer_id.as_str(),
1702 address: descriptor.address.to_string(),
1703 signing_public_key: encode_supervisor_signing_public_key(descriptor.pubkey),
1704 epoch,
1705 },
1706 sender_peer_id,
1707 )
1708 .await
1709 .map_err(|error| format!("{context}: route refresh rejected: {error}"))?;
1710 let obligation =
1711 single_supervisor_publish_obligation(adapter, session_id, &transition, context).await?;
1712 validate_supervisor_publish_obligation(&obligation, descriptor, epoch, context)?;
1713 publish_supervisor_trust_from_generated_obligation(
1714 adapter,
1715 session_id,
1716 comms_runtime,
1717 &obligation,
1718 )
1719 .await
1720}
1721
1722async fn republish_private_supervisor_trust(
1733 adapter: &MeerkatMachine,
1734 session_id: &SessionId,
1735 comms_runtime: &dyn CommsRuntime,
1736 supervisor: &TrustedPeerDescriptor,
1737 epoch: u64,
1738 sender_peer_id: Option<String>,
1739) -> Result<(), String> {
1740 refresh_and_publish_bound_supervisor_route(
1741 adapter,
1742 session_id,
1743 comms_runtime,
1744 supervisor,
1745 epoch,
1746 sender_peer_id,
1747 "bind member idempotent ack trust repair",
1748 )
1749 .await
1750}
1751
1752async fn rollback_bind_after_trust_publication_failure(
1753 adapter: &MeerkatMachine,
1754 session_id: &SessionId,
1755 peer_id: &str,
1756 epoch: u64,
1757) -> Result<(), SupervisorBindingStageError> {
1758 let transition = adapter
1759 .stage_supervisor_revoke(session_id, peer_id.to_string(), epoch)
1760 .await?;
1761 if let Some(obligation) =
1762 crate::protocol_supervisor_trust_revoke::extract_obligations(&transition)
1763 .into_iter()
1764 .find(|obligation| {
1765 obligation.peer_id().as_str() == peer_id && obligation.epoch() == epoch
1766 })
1767 {
1768 adapter
1769 .stage_supervisor_trust_revoked(
1770 session_id,
1771 obligation.peer_id().clone(),
1772 obligation.epoch(),
1773 )
1774 .await?;
1775 }
1776 Ok(())
1777}
1778
1779async fn commit_supervisor_bind_with_rollback(
1789 adapter: &MeerkatMachine,
1790 session_id: &SessionId,
1791 comms_runtime: &dyn CommsRuntime,
1792 supervisor_spec: &TrustedPeerDescriptor,
1793 epoch: u64,
1794 context: &str,
1795) -> Result<(), String> {
1796 adapter
1797 .stage_local_endpoint_for_comms_runtime(session_id, comms_runtime)
1798 .await
1799 .map_err(|error| format!("{context} failed: local endpoint rejected: {error}"))?;
1800 let bind_transition = adapter
1804 .stage_supervisor_bind(
1805 session_id,
1806 supervisor_spec.name.as_str().to_owned(),
1807 supervisor_spec.peer_id.as_str(),
1808 supervisor_spec.address.to_string(),
1809 encode_supervisor_signing_public_key(supervisor_spec.pubkey),
1810 epoch,
1811 )
1812 .await
1813 .map_err(|error| format!("{context} failed: DSL rejected binding: {error}"))?;
1814 let publish_obligation =
1815 match single_supervisor_publish_obligation(adapter, session_id, &bind_transition, context)
1816 .await
1817 {
1818 Ok(obligation) => obligation,
1819 Err(error) => {
1820 let _ = rollback_bind_after_trust_publication_failure(
1821 adapter,
1822 session_id,
1823 &supervisor_spec.peer_id.as_str(),
1824 epoch,
1825 )
1826 .await;
1827 return Err(error);
1828 }
1829 };
1830 if let Err(error) =
1831 validate_supervisor_publish_obligation(&publish_obligation, supervisor_spec, epoch, context)
1832 {
1833 let _ = rollback_bind_after_trust_publication_failure(
1834 adapter,
1835 session_id,
1836 publish_obligation.peer_id(),
1837 publish_obligation.epoch(),
1838 )
1839 .await;
1840 return Err(error);
1841 }
1842 if let Err(error) = publish_supervisor_trust_from_generated_obligation(
1843 adapter,
1844 session_id,
1845 comms_runtime,
1846 &publish_obligation,
1847 )
1848 .await
1849 {
1850 let _ = adapter
1851 .stage_supervisor_trust_publish_failed(
1852 session_id,
1853 publish_obligation.peer_id().clone(),
1854 publish_obligation.epoch(),
1855 error.clone(),
1856 )
1857 .await;
1858 let reason = match rollback_bind_after_trust_publication_failure(
1859 adapter,
1860 session_id,
1861 publish_obligation.peer_id(),
1862 publish_obligation.epoch(),
1863 )
1864 .await
1865 {
1866 Ok(()) => {
1867 format!("{context} failed: trust publication failed after DSL commit: {error}")
1868 }
1869 Err(rollback_error) => format!(
1870 "{context} failed: trust publication failed after DSL commit: {error}; rollback failed: {rollback_error}"
1871 ),
1872 };
1873 return Err(reason);
1874 }
1875 Ok(())
1876}
1877
1878#[derive(Debug, thiserror::Error)]
1884pub enum SupervisorMaterializeBindError {
1885 #[error("supervisor bind admission failed: {0}")]
1888 Admission(String),
1889 #[error("supervisor bind admission rejected: {0}")]
1891 AdmissionRejected(String),
1892 #[error("supervisor trust repair failed for idempotent rebind: {0}")]
1895 TrustRepair(String),
1896 #[error("supervisor bind commit failed: {0}")]
1899 Commit(String),
1900}
1901
1902pub async fn bind_supervisor_for_materialized_session(
1919 adapter: &MeerkatMachine,
1920 session_id: &SessionId,
1921 comms_runtime: &dyn CommsRuntime,
1922 supervisor: &TrustedPeerDescriptor,
1923 epoch: u64,
1924) -> Result<(), SupervisorMaterializeBindError> {
1925 let admission = adapter
1926 .resolve_supervisor_bind_admission(
1927 session_id,
1928 supervisor.peer_id.as_str(),
1929 epoch,
1930 Some(supervisor.peer_id.as_str()),
1934 )
1935 .await
1936 .map_err(|error| SupervisorMaterializeBindError::Admission(error.to_string()))?;
1937 match admission {
1938 SupervisorBindAdmission::Bootstrap => commit_supervisor_bind_with_rollback(
1939 adapter,
1940 session_id,
1941 comms_runtime,
1942 supervisor,
1943 epoch,
1944 "materialized member supervisor bind",
1945 )
1946 .await
1947 .map_err(SupervisorMaterializeBindError::Commit),
1948 SupervisorBindAdmission::IdempotentAck => republish_private_supervisor_trust(
1949 adapter,
1950 session_id,
1951 comms_runtime,
1952 supervisor,
1953 epoch,
1954 Some(supervisor.peer_id.as_str()),
1955 )
1956 .await
1957 .map_err(SupervisorMaterializeBindError::TrustRepair),
1958 SupervisorBindAdmission::Rejected(rejection) => Err(
1959 SupervisorMaterializeBindError::AdmissionRejected(format!("{rejection:?}")),
1960 ),
1961 }
1962}
1963
1964#[derive(Debug, thiserror::Error)]
1967pub enum SupervisorMaterializeAuthorizeError {
1968 #[error("supervisor authorize admission rejected: {0}")]
1971 AdmissionRejected(String),
1972 #[error("supervisor authorize apply failed: {0}")]
1975 Apply(String),
1976}
1977
1978pub async fn authorize_supervisor_for_materialized_session(
1985 adapter: &MeerkatMachine,
1986 session_id: &SessionId,
1987 comms_runtime: &dyn CommsRuntime,
1988 supervisor: &TrustedPeerDescriptor,
1989 epoch: u64,
1990) -> Result<(), SupervisorMaterializeAuthorizeError> {
1991 apply_supervisor_authorization(
1992 adapter,
1993 session_id,
1994 comms_runtime,
1995 supervisor,
1996 epoch,
1997 Some(supervisor.peer_id.as_str()),
1998 Some(encode_supervisor_signing_public_key(supervisor.pubkey)),
1999 "materialized member supervisor authorize",
2000 )
2001 .await
2002 .map_err(|error| match error {
2003 SupervisorAuthorizeApplyError::Rejected(rejection) => {
2004 SupervisorMaterializeAuthorizeError::AdmissionRejected(format!("{rejection:?}"))
2005 }
2006 SupervisorAuthorizeApplyError::Internal(reason) => {
2007 SupervisorMaterializeAuthorizeError::Apply(reason)
2008 }
2009 })
2010}
2011
2012#[cfg(test)]
2013async fn register_member_incarnation(
2014 adapter: &Arc<MeerkatMachine>,
2015 session_id: SessionId,
2016 incarnation: meerkat_contracts::wire::supervisor_bridge::BridgeMemberIncarnation,
2017) -> Result<(), crate::traits::RuntimeDriverError> {
2018 let update = adapter.begin_member_residency_update(session_id).await;
2019 let publication = update.commit(incarnation, None).await?;
2020 drop(publication);
2021 Ok(())
2022}
2023
2024fn require_registered_member_incarnation(
2029 adapter: &MeerkatMachine,
2030 session_id: &SessionId,
2031 expected: &BridgeMemberIncarnation,
2032 context: &str,
2033) -> Result<(), (BridgeRejectionCause, String)> {
2034 match adapter.member_incarnation(session_id) {
2035 Some(current) if ¤t == expected => Ok(()),
2036 Some(current) => Err((
2037 BridgeRejectionCause::StaleFence,
2038 format!("{context}: expected member incarnation {expected:?}; current is {current:?}"),
2039 )),
2040 None => Err((
2041 BridgeRejectionCause::Unavailable,
2042 format!(
2043 "{context}: target session '{session_id}' has no registered host-member incarnation"
2044 ),
2045 )),
2046 }
2047}
2048
2049#[cfg(test)]
2050fn peer_recipient_incarnation_matches(
2051 expected: &meerkat_core::comms::PeerRecipientIncarnation,
2052 current: &BridgeMemberIncarnation,
2053) -> bool {
2054 expected.mob_id == current.mob_id
2055 && expected.agent_identity == current.agent_identity
2056 && expected.host_id == current.host_id
2057 && expected.binding_generation == current.binding_generation
2058 && expected.member_session_id == current.member_session_id
2059 && expected.generation == current.generation
2060 && expected.fence_token == current.fence_token
2061}
2062
2063fn require_optional_registered_member_incarnation(
2064 adapter: &MeerkatMachine,
2065 session_id: &SessionId,
2066 expected: Option<&BridgeMemberIncarnation>,
2067 context: &str,
2068) -> Result<(), (BridgeRejectionCause, String)> {
2069 match expected {
2070 Some(expected) => {
2071 require_registered_member_incarnation(adapter, session_id, expected, context)
2072 }
2073 None if adapter.member_incarnation(session_id).is_some() => Err((
2074 BridgeRejectionCause::StaleFence,
2075 format!(
2076 "{context}: host-materialized target requires an exact expected_member incarnation"
2077 ),
2078 )),
2079 None => Ok(()),
2080 }
2081}
2082
2083async fn acquire_registered_member_live_authority(
2084 adapter: &MeerkatMachine,
2085 session_id: &SessionId,
2086 expected: &BridgeMemberIncarnation,
2087 context: &str,
2088) -> Result<crate::meerkat_machine::MemberLiveAuthorityGuard, (BridgeRejectionCause, String)> {
2089 adapter
2090 .lock_member_live_authority(session_id, expected)
2091 .await
2092 .map_err(|error| match error {
2093 crate::traits::RuntimeDriverError::StaleAuthority { reason } => (
2094 BridgeRejectionCause::StaleFence,
2095 format!("{context}: {reason}"),
2096 ),
2097 other => (
2098 BridgeRejectionCause::Unavailable,
2099 format!("{context}: failed to acquire member-live authority: {other}"),
2100 ),
2101 })
2102}
2103
2104async fn acquire_optional_member_effect_authority(
2105 adapter: &MeerkatMachine,
2106 session_id: &SessionId,
2107 expected: Option<&BridgeMemberIncarnation>,
2108 context: &str,
2109) -> Result<crate::meerkat_machine::MemberEffectAuthorityGuard, (BridgeRejectionCause, String)> {
2110 adapter
2111 .lock_optional_member_effect_authority(session_id, expected)
2112 .await
2113 .map_err(|error| match error {
2114 crate::traits::RuntimeDriverError::StaleAuthority { reason } => (
2115 BridgeRejectionCause::StaleFence,
2116 format!("{context}: {reason}"),
2117 ),
2118 other => (
2119 BridgeRejectionCause::Unavailable,
2120 format!("{context}: failed to acquire member effect authority: {other}"),
2121 ),
2122 })
2123}
2124
2125const MEMBER_POLL_WAIT_MAX_MS: u32 = 20_000;
2129
2130const MEMBER_POLL_DEFAULT_MAX_ROWS: u32 = 256;
2132const MEMBER_POLL_MAX_ROWS: u32 = 1024;
2133
2134const MEMBER_POLL_DEFAULT_MAX_OUTCOMES: u32 = 8;
2137const MEMBER_POLL_MAX_OUTCOMES: u32 = 64;
2138const MEMBER_EVENTS_REPLY_MAX_BYTES: usize = 512 * 1024;
2144const MEMBER_HISTORY_REPLY_MAX_BYTES: usize = 512 * 1024;
2145
2146const HARD_CANCEL_SETTLE_TIMEOUT: Duration = Duration::from_secs(4);
2151const HARD_CANCEL_REASSERT_INTERVAL: Duration = Duration::from_millis(10);
2152
2153fn observation_error_to_bridge_rejection(
2155 error: &MemberObservationError,
2156) -> (BridgeRejectionCause, String) {
2157 match error {
2158 MemberObservationError::StaleIncarnation { .. } => {
2159 (BridgeRejectionCause::StaleFence, error.to_string())
2160 }
2161 MemberObservationError::StaleCursor {
2162 watermark,
2163 generation,
2164 } => (
2165 BridgeRejectionCause::StaleCursor {
2166 watermark: *watermark,
2167 generation: *generation,
2168 },
2169 error.to_string(),
2170 ),
2171 MemberObservationError::FutureGenerationCursor { .. } => {
2174 (BridgeRejectionCause::Internal, error.to_string())
2175 }
2176 MemberObservationError::Unavailable { .. } => {
2177 (BridgeRejectionCause::Unavailable, error.to_string())
2178 }
2179 MemberObservationError::OutcomeRecordTooLarge { .. } => {
2180 (BridgeRejectionCause::Internal, error.to_string())
2181 }
2182 MemberObservationError::Internal { .. } => {
2183 (BridgeRejectionCause::Internal, error.to_string())
2184 }
2185 }
2186}
2187
2188fn resolve_member_observation_host(
2192 adapter: &Arc<MeerkatMachine>,
2193) -> Result<Arc<dyn MemberObservationHost>, (BridgeRejectionCause, String)> {
2194 adapter.member_observation_host().ok_or_else(|| {
2195 (
2196 BridgeRejectionCause::Unavailable,
2197 "member host serves no observation substrate".to_string(),
2198 )
2199 })
2200}
2201
2202fn resolve_member_live_host(
2208 adapter: &Arc<MeerkatMachine>,
2209) -> Result<Arc<dyn MemberLiveHost>, (BridgeRejectionCause, String)> {
2210 adapter.note_live_command_served();
2214 adapter.member_live_host().ok_or_else(|| {
2215 (
2216 BridgeRejectionCause::LiveTransportUnavailable,
2217 "member host serves no live substrate".to_string(),
2218 )
2219 })
2220}
2221
2222fn member_live_error_to_bridge_rejection(
2227 error: &MemberLiveError,
2228) -> (BridgeRejectionCause, String) {
2229 (error.to_bridge_rejection(), error.to_string())
2230}
2231
2232async fn stage_correlated_reply_endpoint(
2241 comms_runtime: &Arc<dyn CommsRuntime>,
2242 candidate: &PeerInputCandidate,
2243) {
2244 if let (Some(endpoint), Some(sender_peer_id), Some(signing_pubkey)) = (
2245 candidate.ingress.declared_reply_endpoint.clone(),
2246 candidate.ingress.canonical_peer_id,
2247 candidate.ingress.signing_pubkey,
2248 ) {
2249 match comms_runtime
2250 .stage_correlated_reply_endpoint(
2251 sender_peer_id,
2252 candidate.interaction.id,
2253 signing_pubkey,
2254 endpoint,
2255 )
2256 .await
2257 {
2258 Ok(()) | Err(SendError::Unsupported(_)) => {}
2259 Err(error) => {
2260 tracing::warn!(
2261 interaction_id = %candidate.interaction.id,
2262 error = %error,
2263 "comms_drain: failed to stage authenticated correlated reply endpoint"
2264 );
2265 }
2266 }
2267 }
2268}
2269
2270async fn send_bridge_response(
2271 comms_runtime: &Arc<dyn CommsRuntime>,
2272 candidate: &PeerInputCandidate,
2273 status: meerkat_core::interaction::ResponseStatus,
2274 reply: BridgeReply,
2275 _declared_reply_address: Option<&str>,
2279) {
2280 let result = serde_json::to_value(&reply).unwrap_or_else(|error| {
2281 tracing::error!(
2282 interaction_id = %candidate.interaction.id,
2283 error = %error,
2284 "comms_drain: BridgeReply serialization failed; falling back to minimal rejection"
2285 );
2286 serde_json::json!({
2287 "result": "rejected",
2288 "cause": "internal",
2289 "reason": "bridge reply serialization failed",
2290 })
2291 });
2292 let to = match resolve_bridge_response_route(comms_runtime, candidate).await {
2293 Some(route) => route,
2294 None => {
2295 tracing::warn!(
2296 from = %candidate.ingress.diagnostic_label(),
2297 interaction_id = %candidate.interaction.id,
2298 "comms_drain: failed to resolve bridge response peer route"
2299 );
2300 comms_runtime.mark_interaction_complete(&candidate.interaction.id);
2301 return;
2302 }
2303 };
2304
2305 stage_correlated_reply_endpoint(comms_runtime, candidate).await;
2306
2307 if let Err(error) = comms_runtime
2308 .send(CommsCommand::PeerResponse {
2309 to,
2310 in_reply_to: candidate.interaction.id,
2311 status,
2312 result,
2313 blocks: None,
2314 content_taint: None,
2315 handling_mode: None,
2316 objective_id: candidate.interaction.objective_id,
2317 })
2318 .await
2319 {
2320 if let Some(sender_peer_id) = candidate.ingress.canonical_peer_id
2321 && candidate.ingress.declared_reply_endpoint.is_some()
2322 && let Err(cleanup_error) = comms_runtime
2323 .unstage_correlated_reply_endpoint(sender_peer_id, candidate.interaction.id)
2324 .await
2325 && !matches!(cleanup_error, SendError::Unsupported(_))
2326 {
2327 tracing::warn!(
2328 from = %candidate.ingress.diagnostic_label(),
2329 interaction_id = %candidate.interaction.id,
2330 error = %cleanup_error,
2331 "comms_drain: failed to clear correlated reply endpoint after response failure"
2332 );
2333 }
2334 tracing::warn!(
2335 from = %candidate.ingress.diagnostic_label(),
2336 interaction_id = %candidate.interaction.id,
2337 error = %error,
2338 "comms_drain: failed to send bridge response"
2339 );
2340 }
2341 comms_runtime.mark_interaction_complete(&candidate.interaction.id);
2342}
2343
2344async fn resolve_bridge_response_route(
2345 comms_runtime: &Arc<dyn CommsRuntime>,
2346 candidate: &PeerInputCandidate,
2347) -> Option<PeerRoute> {
2348 if let Some(sender_route) = candidate.ingress.route.clone() {
2349 if let Some(route) = resolve_peer_route(comms_runtime, sender_route.peer_id).await {
2350 return Some(route);
2351 }
2352 return Some(sender_route);
2353 }
2354
2355 if let Some(sender_peer_id) = candidate.ingress.canonical_peer_id {
2356 return resolve_peer_route(comms_runtime, sender_peer_id)
2357 .await
2358 .or_else(|| Some(PeerRoute::new(sender_peer_id)));
2359 }
2360 None
2361}
2362
2363async fn resolve_peer_route(
2364 comms_runtime: &Arc<dyn CommsRuntime>,
2365 peer_id: PeerId,
2366) -> Option<PeerRoute> {
2367 let peers = comms_runtime.peers().await;
2368 peers
2369 .iter()
2370 .find(|entry| entry.peer_id == peer_id)
2371 .map(|entry| PeerRoute::with_display_name(entry.peer_id, entry.name.clone()))
2372}
2373
2374fn record_bridge_inbound_peer_request(
2375 comms_runtime: &Arc<dyn CommsRuntime>,
2376 candidate: &PeerInputCandidate,
2377) -> bool {
2378 let Some(handle) = comms_runtime.peer_request_response_authority_handle() else {
2379 tracing::warn!(
2380 interaction_id = %candidate.interaction.id,
2381 "comms_drain: rejected supervisor bridge request without complete peer request authority"
2382 );
2383 comms_runtime.mark_interaction_complete(&candidate.interaction.id);
2384 return false;
2385 };
2386 let corr_id = meerkat_core::PeerCorrelationId::from_uuid(candidate.interaction.id.0);
2387 if handle.inbound_state(corr_id).is_some() {
2388 return true;
2389 }
2390 if let Err(err) = handle.request_received(corr_id, candidate.interaction.handling_mode) {
2391 tracing::warn!(
2392 error = %err,
2393 corr_id = %corr_id,
2394 "PeerInteractionHandle::request_received rejected for supervisor bridge command"
2395 );
2396 comms_runtime.mark_interaction_complete(&candidate.interaction.id);
2397 return false;
2398 }
2399 true
2400}
2401
2402async fn send_bridge_failure(
2403 comms_runtime: &Arc<dyn CommsRuntime>,
2404 candidate: &PeerInputCandidate,
2405 cause: BridgeRejectionCause,
2406 message: impl Into<String>,
2407 declared_reply_address: Option<&str>,
2408) {
2409 send_bridge_response(
2410 comms_runtime,
2411 candidate,
2412 meerkat_core::interaction::ResponseStatus::Failed,
2413 BridgeReply::Rejected {
2414 cause,
2415 reason: message.into(),
2416 },
2417 declared_reply_address,
2418 )
2419 .await;
2420}
2421
2422async fn ensure_bridge_response_route_for_supervisor(
2423 adapter: &Arc<MeerkatMachine>,
2424 session_id: &SessionId,
2425 comms_runtime: &Arc<dyn CommsRuntime>,
2426 supervisor: &BridgePeerIdentity,
2427 context: &str,
2428) -> Result<(), (BridgeRejectionCause, String)> {
2429 ensure_bridge_response_route_for_descriptor(
2430 adapter,
2431 session_id,
2432 comms_runtime,
2433 supervisor.clone().into_trusted_peer_descriptor(),
2434 context,
2435 )
2436 .await
2437}
2438
2439async fn ensure_bridge_response_route_for_descriptor(
2440 adapter: &Arc<MeerkatMachine>,
2441 session_id: &SessionId,
2442 comms_runtime: &Arc<dyn CommsRuntime>,
2443 descriptor: TrustedPeerDescriptor,
2444 context: &str,
2445) -> Result<(), (BridgeRejectionCause, String)> {
2446 let obligation = supervisor_route_publish_obligation(adapter, session_id, &descriptor, context)
2447 .await
2448 .map_err(|reason| (BridgeRejectionCause::Internal, reason))?;
2449 let authority = supervisor_publish_authority(&obligation)
2450 .map_err(|reason| (BridgeRejectionCause::Internal, reason))?;
2451 apply_generated_private_trust_add(comms_runtime.as_ref(), descriptor, authority)
2452 .await
2453 .map_err(|error| {
2454 (
2455 BridgeRejectionCause::Internal,
2456 format!("{context}: failed to install supervisor response route: {error}"),
2457 )
2458 })
2459}
2460
2461async fn supervisor_route_publish_obligation(
2467 adapter: &MeerkatMachine,
2468 session_id: &SessionId,
2469 descriptor: &TrustedPeerDescriptor,
2470 context: &str,
2471) -> Result<crate::protocol_supervisor_trust_publish::SupervisorTrustPublishObligation, String> {
2472 let binding = adapter.supervisor_binding(session_id).await;
2473 let bound = BoundSupervisorState::from_binding(&binding).ok_or_else(|| {
2474 format!("{context}: supervisor binding absent for response-route authority")
2475 })?;
2476 let transition = adapter
2477 .stage_supervisor_trust_publish_request(
2478 session_id,
2479 bound.name.clone(),
2480 bound.peer_id.clone(),
2481 bound.address.clone(),
2482 bound.signing_public_key.clone(),
2483 bound.epoch,
2484 )
2485 .await
2486 .map_err(|error| {
2487 format!("{context}: generated supervisor route publish request rejected: {error}")
2488 })?;
2489 let obligation =
2490 single_supervisor_publish_obligation(adapter, session_id, &transition, context).await?;
2491 if obligation.peer_id().as_str() != descriptor.peer_id.as_str() {
2492 return Err(format!(
2493 "{context}: generated supervisor route publish obligation peer_id mismatch"
2494 ));
2495 }
2496 Ok(obligation)
2497}
2498
2499struct SupervisorRotationDriveContext<'a> {
2514 operation_id: &'a str,
2515 previous: &'a BoundSupervisorState,
2516 next: &'a TrustedPeerDescriptor,
2517 next_epoch: u64,
2518 phase: crate::meerkat_machine::dsl::SupervisorRotationPhase,
2519}
2520
2521async fn drive_supervisor_rotation(
2522 adapter: &Arc<MeerkatMachine>,
2523 session_id: &SessionId,
2524 comms_runtime: &Arc<dyn CommsRuntime>,
2525 context: SupervisorRotationDriveContext<'_>,
2526) -> Result<(), String> {
2527 use crate::meerkat_machine::dsl::SupervisorRotationPhase;
2528 let SupervisorRotationDriveContext {
2529 operation_id,
2530 previous,
2531 next,
2532 next_epoch,
2533 mut phase,
2534 } = context;
2535
2536 if matches!(
2537 phase,
2538 SupervisorRotationPhase::Completed | SupervisorRotationPhase::Rejected
2539 ) {
2540 return Ok(());
2541 }
2542 let mut transition = adapter
2543 .stage_supervisor_rotation_resume(session_id, operation_id.to_string())
2544 .await
2545 .map_err(|error| format!("rotation resume rejected: {error}"))?;
2546
2547 loop {
2548 match phase {
2549 SupervisorRotationPhase::PreviousRevokePending => {
2550 let obligation = matching_supervisor_revoke_obligation(
2551 adapter,
2552 session_id,
2553 &transition,
2554 &previous.peer_id,
2555 previous.epoch,
2556 "supervisor rotation operation",
2557 )
2558 .await?;
2559 let authority = supervisor_revoke_authority(&obligation)?;
2560 let removal = crate::tokio::time::timeout(
2561 SUPERVISOR_RESPONSE_ROUTE_IO_TIMEOUT,
2562 apply_generated_private_trust_remove(
2563 comms_runtime.as_ref(),
2564 obligation.peer_id().clone(),
2565 authority,
2566 ),
2567 )
2568 .await;
2569 if let Err(error) = match removal {
2570 Ok(result) => result.map(|_| ()),
2571 Err(_) => Err("previous supervisor trust removal timed out".to_string()),
2572 } {
2573 let _ = adapter
2574 .stage_supervisor_trust_revoke_failed(
2575 session_id,
2576 obligation.peer_id().clone(),
2577 obligation.epoch(),
2578 error.clone(),
2579 )
2580 .await;
2581 return Err(format!(
2582 "supervisor rotation operation left previous revoke pending: {error}"
2583 ));
2584 }
2585 transition = adapter
2586 .stage_supervisor_rotation_previous_revoked(
2587 session_id,
2588 operation_id.to_string(),
2589 obligation.peer_id().clone(),
2590 obligation.epoch(),
2591 )
2592 .await
2593 .map_err(|error| {
2594 format!(
2595 "previous supervisor route is absent but rotation checkpoint persistence failed: {error}"
2596 )
2597 })?;
2598 phase = SupervisorRotationPhase::NextPublishPending;
2599 }
2600 SupervisorRotationPhase::NextPublishPending => {
2601 let obligation = single_supervisor_publish_obligation(
2602 adapter,
2603 session_id,
2604 &transition,
2605 "supervisor rotation operation",
2606 )
2607 .await?;
2608 validate_supervisor_publish_obligation(
2609 &obligation,
2610 next,
2611 next_epoch,
2612 "supervisor rotation operation",
2613 )?;
2614 let authority = supervisor_publish_authority(&obligation)?;
2615 let publication = crate::tokio::time::timeout(
2616 SUPERVISOR_RESPONSE_ROUTE_IO_TIMEOUT,
2617 apply_generated_private_trust_add(
2618 comms_runtime.as_ref(),
2619 next.clone(),
2620 authority,
2621 ),
2622 )
2623 .await;
2624 if let Err(error) = match publication {
2625 Ok(result) => result,
2626 Err(_) => Err("next supervisor trust publication timed out".to_string()),
2627 } {
2628 let _ = adapter
2629 .stage_supervisor_trust_publish_failed(
2630 session_id,
2631 obligation.peer_id().clone(),
2632 obligation.epoch(),
2633 error.clone(),
2634 )
2635 .await;
2636 return Err(format!(
2637 "supervisor rotation operation left next publication pending: {error}"
2638 ));
2639 }
2640 adapter
2645 .stage_supervisor_rotation_next_published(
2646 session_id,
2647 operation_id.to_string(),
2648 obligation.peer_id().clone(),
2649 obligation.epoch(),
2650 )
2651 .await
2652 .map_err(|error| {
2653 format!(
2654 "next supervisor route is live but rotation checkpoint persistence failed: {error}"
2655 )
2656 })?;
2657 phase = SupervisorRotationPhase::Completed;
2658 }
2659 SupervisorRotationPhase::Completed | SupervisorRotationPhase::Rejected => return Ok(()),
2660 }
2661 }
2662}
2663
2664fn trusted_peer_from_generated_binding(
2665 binding: &crate::meerkat_machine::GeneratedSupervisorBinding,
2666) -> Result<TrustedPeerDescriptor, String> {
2667 TrustedPeerDescriptor::unsigned_with_pubkey(
2668 binding.name.clone(),
2669 &binding.peer_id,
2670 decode_supervisor_signing_public_key(&binding.signing_public_key)?,
2671 &binding.address,
2672 )
2673}
2674
2675async fn start_supervisor_rotation_worker_if_pending(
2676 adapter: &Arc<MeerkatMachine>,
2677 session_id: &SessionId,
2678 comms_runtime: &Arc<dyn CommsRuntime>,
2679) -> Result<bool, String> {
2680 let Some(receipt) = adapter
2681 .active_supervisor_rotation(session_id)
2682 .await
2683 .map_err(|error| error.to_string())?
2684 else {
2685 return Ok(false);
2686 };
2687 if matches!(
2688 receipt.phase,
2689 crate::meerkat_machine::dsl::SupervisorRotationPhase::Completed
2690 | crate::meerkat_machine::dsl::SupervisorRotationPhase::Rejected
2691 ) {
2692 return Ok(false);
2693 }
2694 let operation_id = receipt.operation_id.clone();
2695 let worker_adapter = Arc::clone(adapter);
2696 let worker_session_id = session_id.clone();
2697 let worker_runtime = Arc::clone(comms_runtime);
2698 let slot_runtime = Arc::clone(comms_runtime);
2699 let task_operation_id = operation_id.clone();
2700 let (start_tx, start_rx) = crate::tokio::sync::oneshot::channel::<()>();
2701 let task = crate::tokio::spawn(async move {
2702 if start_rx.await.is_err() {
2703 return;
2704 }
2705 let mut retry_delay = std::time::Duration::from_millis(25);
2706 loop {
2707 match worker_adapter
2708 .peer_ingress_runtime_is_current(&worker_session_id, &worker_runtime)
2709 .await
2710 {
2711 Ok(true) => {}
2712 Ok(false) => {
2713 tracing::warn!(
2714 session_id = %worker_session_id,
2715 operation_id = %task_operation_id,
2716 "supervisor rotation worker refused stale runtime retry"
2717 );
2718 return;
2719 }
2720 Err(error) => {
2721 tracing::warn!(
2722 session_id = %worker_session_id,
2723 operation_id = %task_operation_id,
2724 %error,
2725 "supervisor rotation worker lost runtime authority before retry"
2726 );
2727 return;
2728 }
2729 }
2730 let receipt = match worker_adapter
2731 .active_supervisor_rotation(&worker_session_id)
2732 .await
2733 {
2734 Ok(Some(receipt)) if receipt.operation_id == task_operation_id => receipt,
2735 Ok(_) => return,
2736 Err(error) => {
2737 tracing::warn!(
2738 session_id = %worker_session_id,
2739 operation_id = %task_operation_id,
2740 %error,
2741 "supervisor rotation worker lost its registered session"
2742 );
2743 return;
2744 }
2745 };
2746 if matches!(
2747 receipt.phase,
2748 crate::meerkat_machine::dsl::SupervisorRotationPhase::Completed
2749 | crate::meerkat_machine::dsl::SupervisorRotationPhase::Rejected
2750 ) {
2751 return;
2752 }
2753 let next = match trusted_peer_from_generated_binding(&receipt.next) {
2754 Ok(next) => next,
2755 Err(error) => {
2756 tracing::error!(
2757 session_id = %worker_session_id,
2758 operation_id = %task_operation_id,
2759 %error,
2760 "supervisor rotation receipt carried an invalid target"
2761 );
2762 return;
2763 }
2764 };
2765 let previous = BoundSupervisorState {
2766 name: receipt.previous.name,
2767 peer_id: receipt.previous.peer_id,
2768 address: receipt.previous.address,
2769 signing_public_key: receipt.previous.signing_public_key,
2770 epoch: receipt.previous.epoch,
2771 };
2772 match drive_supervisor_rotation(
2773 &worker_adapter,
2774 &worker_session_id,
2775 &worker_runtime,
2776 SupervisorRotationDriveContext {
2777 operation_id: &task_operation_id,
2778 previous: &previous,
2779 next: &next,
2780 next_epoch: receipt.next.epoch,
2781 phase: receipt.phase,
2782 },
2783 )
2784 .await
2785 {
2786 Ok(()) => return,
2787 Err(error) => {
2788 tracing::warn!(
2789 session_id = %worker_session_id,
2790 operation_id = %task_operation_id,
2791 %error,
2792 ?retry_delay,
2793 "supervisor rotation remains pending; session-owned worker will retry"
2794 );
2795 crate::tokio::time::sleep(retry_delay).await;
2796 retry_delay = std::cmp::min(
2797 retry_delay.saturating_mul(2),
2798 std::time::Duration::from_secs(1),
2799 );
2800 }
2801 }
2802 }
2803 });
2804 let installed = adapter
2805 .install_supervisor_rotation_task_if_current(session_id, operation_id, slot_runtime, task)
2806 .await
2807 .map_err(|error| error.to_string())?;
2808 if installed {
2809 start_tx
2810 .send(())
2811 .map_err(|()| "supervisor rotation worker exited before start".to_string())?;
2812 }
2813 Ok(installed)
2814}
2815
2816async fn hydrate_current_bound_supervisor_route(
2817 adapter: &Arc<MeerkatMachine>,
2818 session_id: &SessionId,
2819 comms_runtime: &Arc<dyn CommsRuntime>,
2820) -> Result<bool, String> {
2821 let binding = adapter.supervisor_binding(session_id).await;
2822 let Some(bound) = BoundSupervisorState::from_binding(&binding) else {
2823 return Ok(false);
2824 };
2825 let descriptor = TrustedPeerDescriptor::unsigned_with_pubkey(
2826 bound.name,
2827 &bound.peer_id,
2828 decode_supervisor_signing_public_key(&bound.signing_public_key)?,
2829 &bound.address,
2830 )?;
2831 ensure_bridge_response_route_for_descriptor(
2832 adapter,
2833 session_id,
2834 comms_runtime,
2835 descriptor,
2836 "supervisor runtime hydration",
2837 )
2838 .await
2839 .map_err(|(_, reason)| reason)?;
2840 Ok(true)
2841}
2842
2843async fn hydrate_or_resume_supervisor_rotation_runtime(
2844 adapter: &Arc<MeerkatMachine>,
2845 session_id: &SessionId,
2846 comms_runtime: &Arc<dyn CommsRuntime>,
2847) -> Result<(), String> {
2848 match adapter
2849 .active_supervisor_rotation(session_id)
2850 .await
2851 .map_err(|error| error.to_string())?
2852 .map(|receipt| receipt.phase)
2853 {
2854 Some(
2855 crate::meerkat_machine::dsl::SupervisorRotationPhase::PreviousRevokePending
2856 | crate::meerkat_machine::dsl::SupervisorRotationPhase::NextPublishPending,
2857 ) => {
2858 start_supervisor_rotation_worker_if_pending(adapter, session_id, comms_runtime).await?;
2859 }
2860 Some(
2861 crate::meerkat_machine::dsl::SupervisorRotationPhase::Completed
2862 | crate::meerkat_machine::dsl::SupervisorRotationPhase::Rejected,
2863 )
2864 | None => {
2865 let _ =
2871 hydrate_current_bound_supervisor_route(adapter, session_id, comms_runtime).await?;
2872 }
2873 }
2874 Ok(())
2875}
2876
2877fn bridge_supervisor_rotation_state(
2878 receipt: crate::meerkat_machine::GeneratedSupervisorRotationReceipt,
2879) -> Result<BridgeSupervisorRotationState, String> {
2880 let operation_id = receipt
2881 .operation_id
2882 .parse()
2883 .map_err(|error| format!("stored supervisor rotation operation id is invalid: {error}"))?;
2884 let target = BridgePeerSpec {
2885 name: receipt.next.name,
2886 peer_id: receipt.next.peer_id,
2887 address: receipt.next.address,
2888 pubkey: decode_supervisor_signing_public_key(&receipt.next.signing_public_key)?,
2889 };
2890 let operation = BridgeSupervisorRotationOperationReceipt {
2891 operation_id,
2892 target: BridgeSupervisorRotationTargetReceipt {
2893 target,
2894 target_epoch: receipt.next.epoch,
2895 },
2896 };
2897 match receipt.phase {
2898 crate::meerkat_machine::dsl::SupervisorRotationPhase::PreviousRevokePending => {
2899 Ok(BridgeSupervisorRotationState::Pending {
2900 operation,
2901 phase: BridgeSupervisorRotationPendingPhase::PreviousRevokePending,
2902 })
2903 }
2904 crate::meerkat_machine::dsl::SupervisorRotationPhase::NextPublishPending => {
2905 Ok(BridgeSupervisorRotationState::Pending {
2906 operation,
2907 phase: BridgeSupervisorRotationPendingPhase::NextPublishPending,
2908 })
2909 }
2910 crate::meerkat_machine::dsl::SupervisorRotationPhase::Completed => {
2911 Ok(BridgeSupervisorRotationState::Completed { receipt: operation })
2912 }
2913 crate::meerkat_machine::dsl::SupervisorRotationPhase::Rejected => {
2914 let rejection = receipt.rejection.ok_or_else(|| {
2915 "stored rejected supervisor rotation omitted its typed cause".to_string()
2916 })?;
2917 let cause = match rejection {
2918 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::OperationConflict => {
2919 BridgeSupervisorRotationRejectionCause::OperationConflict
2920 }
2921 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::NotBound => {
2922 BridgeSupervisorRotationRejectionCause::Internal
2923 }
2924 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::SenderMismatch => {
2925 BridgeSupervisorRotationRejectionCause::SenderMismatch
2926 }
2927 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::TargetEpochNotAdvanced => {
2928 BridgeSupervisorRotationRejectionCause::StaleTargetEpoch
2929 }
2930 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::InvalidTarget => {
2931 BridgeSupervisorRotationRejectionCause::InvalidTarget
2932 }
2933 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::UnsupportedProtocolVersion => {
2934 BridgeSupervisorRotationRejectionCause::UnsupportedProtocolVersion
2935 }
2936 };
2937 Ok(BridgeSupervisorRotationState::Rejected {
2938 receipt: BridgeSupervisorRotationRejectionReceipt {
2939 operation,
2940 cause,
2941 reason: format!("supervisor rotation rejected: {rejection:?}"),
2942 },
2943 })
2944 }
2945 }
2946}
2947
2948#[derive(Debug)]
2949enum SupervisorAuthorizeApplyError {
2950 Rejected(crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind),
2951 Internal(String),
2952}
2953
2954#[allow(clippy::too_many_arguments)]
2958async fn apply_supervisor_authorization(
2959 adapter: &MeerkatMachine,
2960 session_id: &SessionId,
2961 comms_runtime: &dyn CommsRuntime,
2962 supervisor: &TrustedPeerDescriptor,
2963 epoch: u64,
2964 sender_peer_id: Option<String>,
2965 sender_signing_public_key: Option<String>,
2966 context: &str,
2967) -> Result<(), SupervisorAuthorizeApplyError> {
2968 adapter
2969 .stage_local_endpoint_for_comms_runtime(session_id, comms_runtime)
2970 .await
2971 .map_err(|error| {
2972 SupervisorAuthorizeApplyError::Internal(format!(
2973 "{context}: local endpoint rejected: {error}"
2974 ))
2975 })?;
2976
2977 let admission = adapter
2978 .resolve_supervisor_authorize_admission(
2979 session_id,
2980 GeneratedSupervisorBinding {
2981 name: supervisor.name.as_str().to_owned(),
2982 peer_id: supervisor.peer_id.as_str(),
2983 address: supervisor.address.to_string(),
2984 signing_public_key: encode_supervisor_signing_public_key(supervisor.pubkey),
2985 epoch,
2986 },
2987 sender_peer_id.clone(),
2988 sender_signing_public_key,
2989 )
2990 .await
2991 .map_err(|error| {
2992 SupervisorAuthorizeApplyError::Internal(format!(
2993 "{context}: generated admission failed: {error}"
2994 ))
2995 })?;
2996
2997 match admission {
2998 SupervisorAuthorizeAdmission::IdempotentAck => refresh_and_publish_bound_supervisor_route(
2999 adapter,
3000 session_id,
3001 comms_runtime,
3002 supervisor,
3003 epoch,
3004 sender_peer_id,
3005 context,
3006 )
3007 .await
3008 .map_err(SupervisorAuthorizeApplyError::Internal),
3009 SupervisorAuthorizeAdmission::Proceed(_) => {
3010 let transition = adapter
3011 .stage_supervisor_authorize(
3012 session_id,
3013 supervisor.name.as_str().to_owned(),
3014 supervisor.peer_id.as_str(),
3015 supervisor.address.to_string(),
3016 encode_supervisor_signing_public_key(supervisor.pubkey),
3017 epoch,
3018 )
3019 .await
3020 .map_err(|error| {
3021 SupervisorAuthorizeApplyError::Internal(format!(
3022 "{context}: version advance failed: {error}"
3023 ))
3024 })?;
3025 let obligation =
3026 single_supervisor_publish_obligation(adapter, session_id, &transition, context)
3027 .await
3028 .map_err(SupervisorAuthorizeApplyError::Internal)?;
3029 validate_supervisor_publish_obligation(&obligation, supervisor, epoch, context)
3030 .map_err(SupervisorAuthorizeApplyError::Internal)?;
3031 publish_supervisor_trust_from_generated_obligation(
3032 adapter,
3033 session_id,
3034 comms_runtime,
3035 &obligation,
3036 )
3037 .await
3038 .map_err(|error| {
3039 SupervisorAuthorizeApplyError::Internal(format!(
3040 "{context}: version publish failed: {error}"
3041 ))
3042 })
3043 }
3044 SupervisorAuthorizeAdmission::Rejected(rejection) => {
3045 Err(SupervisorAuthorizeApplyError::Rejected(rejection))
3046 }
3047 }
3048}
3049
3050async fn handle_authorize_supervisor_command(
3051 adapter: &Arc<MeerkatMachine>,
3052 session_id: &SessionId,
3053 comms_runtime: &Arc<dyn CommsRuntime>,
3054 candidate: &PeerInputCandidate,
3055 sender: &PeerIngressFact,
3056 payload: BridgeSupervisorPayload,
3057) -> bool {
3058 let supervisor = match bridge_peer_identity(&payload.supervisor, "authorize supervisor failed")
3059 {
3060 Ok(supervisor) => supervisor,
3061 Err((cause, reason)) => {
3062 send_bridge_failure(
3063 comms_runtime,
3064 candidate,
3065 cause,
3066 reason,
3067 Some(payload.supervisor.address.as_str()),
3068 )
3069 .await;
3070 return true;
3071 }
3072 };
3073 let supervisor_spec = match TrustedPeerDescriptor::try_from(payload.supervisor.clone()) {
3074 Ok(supervisor_spec) => supervisor_spec,
3075 Err(error) => {
3076 send_bridge_failure(
3077 comms_runtime,
3078 candidate,
3079 BridgeRejectionCause::InvalidSupervisorSpec,
3080 format!("authorize supervisor failed: invalid supervisor peer spec: {error}"),
3081 Some(payload.supervisor.address.as_str()),
3082 )
3083 .await;
3084 return true;
3085 }
3086 };
3087 match apply_supervisor_authorization(
3088 adapter,
3089 session_id,
3090 comms_runtime.as_ref(),
3091 &supervisor_spec,
3092 payload.epoch,
3093 generated_sender_peer_id(sender),
3094 generated_sender_signing_public_key(sender),
3095 "authorize supervisor",
3096 )
3097 .await
3098 {
3099 Ok(()) => {
3100 if crate::tokio::time::timeout(
3101 SUPERVISOR_RESPONSE_ROUTE_IO_TIMEOUT,
3102 send_bridge_response(
3103 comms_runtime,
3104 candidate,
3105 meerkat_core::interaction::ResponseStatus::Completed,
3106 BridgeReply::Ack(BridgeAck { ok: true }),
3107 Some(payload.supervisor.address.as_str()),
3108 ),
3109 )
3110 .await
3111 .is_err()
3112 {
3113 comms_runtime.mark_interaction_complete(&candidate.interaction.id);
3114 }
3115 }
3116 Err(SupervisorAuthorizeApplyError::Rejected(rejection)) => {
3117 send_bridge_failure(
3118 comms_runtime,
3119 candidate,
3120 supervisor_authorize_admission_rejection_cause(rejection),
3121 supervisor_authorize_admission_rejection_reason(rejection, sender, &supervisor),
3122 Some(payload.supervisor.address.as_str()),
3123 )
3124 .await;
3125 }
3126 Err(SupervisorAuthorizeApplyError::Internal(error)) => {
3127 send_bridge_failure(
3128 comms_runtime,
3129 candidate,
3130 BridgeRejectionCause::Internal,
3131 error,
3132 Some(payload.supervisor.address.as_str()),
3133 )
3134 .await;
3135 }
3136 }
3137 true
3138}
3139
3140#[derive(serde::Deserialize)]
3141#[serde(rename_all = "snake_case")]
3142enum SupervisorDeliveryMarker {
3143 SubmitSupervisorRotation,
3144}
3145
3146#[derive(serde::Deserialize)]
3147#[serde(deny_unknown_fields)]
3148struct RawSupervisorRotationSubmitDelivery {
3149 #[serde(rename = "delivery")]
3150 _delivery: SupervisorDeliveryMarker,
3151 operation_id: SupervisorRotationOperationId,
3152 target: BridgePeerSpec,
3153 target_epoch: u64,
3154 protocol_version: u32,
3155}
3156
3157#[derive(Default)]
3158struct SupervisorDeliveryMarkerProbe {
3159 delivery_key_count: usize,
3160 reserved_rotation_marker_seen: bool,
3161}
3162
3163impl<'de> serde::Deserialize<'de> for SupervisorDeliveryMarkerProbe {
3164 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
3165 where
3166 D: serde::Deserializer<'de>,
3167 {
3168 struct ProbeVisitor;
3169
3170 impl<'de> serde::de::Visitor<'de> for ProbeVisitor {
3171 type Value = SupervisorDeliveryMarkerProbe;
3172
3173 fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
3174 formatter.write_str("a supervisor delivery JSON object")
3175 }
3176
3177 fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
3178 where
3179 A: serde::de::MapAccess<'de>,
3180 {
3181 let mut probe = SupervisorDeliveryMarkerProbe::default();
3182 while let Some(key) = map.next_key::<String>()? {
3183 if key == "delivery" {
3184 probe.delivery_key_count += 1;
3185 let value = map.next_value::<serde_json::Value>()?;
3186 if value.as_str() == Some("submit_supervisor_rotation") {
3187 probe.reserved_rotation_marker_seen = true;
3188 }
3189 } else {
3190 map.next_value::<serde::de::IgnoredAny>()?;
3191 }
3192 }
3193 Ok(probe)
3194 }
3195 }
3196
3197 deserializer.deserialize_map(ProbeVisitor)
3198 }
3199}
3200
3201async fn try_handle_supervisor_bridge_delivery(
3202 adapter: &Arc<MeerkatMachine>,
3203 session_id: &SessionId,
3204 comms_runtime: &Arc<dyn CommsRuntime>,
3205 candidate: &PeerInputCandidate,
3206) -> bool {
3207 let InteractionContent::Message { body, .. } = &candidate.interaction.content else {
3208 return false;
3209 };
3210 let Ok(marker_probe) = serde_json::from_str::<SupervisorDeliveryMarkerProbe>(body) else {
3211 return false;
3212 };
3213 if !marker_probe.reserved_rotation_marker_seen {
3214 return false;
3215 }
3216 if marker_probe.delivery_key_count != 1 {
3217 tracing::warn!(
3218 %session_id,
3219 delivery_key_count = marker_probe.delivery_key_count,
3220 "duplicate/conflicting supervisor rotation delivery markers consumed fail-closed"
3221 );
3222 comms_runtime.mark_interaction_complete(&candidate.interaction.id);
3223 return true;
3224 }
3225 let delivery = match serde_json::from_str::<RawSupervisorRotationSubmitDelivery>(body) {
3226 Ok(delivery) => delivery,
3227 Err(error) => {
3228 tracing::warn!(%session_id, %error, "malformed supervisor rotation delivery consumed fail-closed");
3229 comms_runtime.mark_interaction_complete(&candidate.interaction.id);
3230 return true;
3231 }
3232 };
3233 let sender_peer_id = generated_sender_peer_id(&candidate.ingress);
3234 let sender_signing_public_key = generated_sender_signing_public_key(&candidate.ingress);
3235 let raw_target = delivery.target;
3236 let (next, preflight_rejection) = if BridgeProtocolVersion::try_from(delivery.protocol_version)
3237 .ok()
3238 == Some(BridgeProtocolVersion::V4)
3239 {
3240 match TrustedPeerDescriptor::try_from(raw_target.clone()) {
3241 Ok(target) => (
3242 GeneratedSupervisorBinding {
3243 name: target.name.as_str().to_owned(),
3244 peer_id: target.peer_id.as_str(),
3245 address: target.address.to_string(),
3246 signing_public_key: encode_supervisor_signing_public_key(target.pubkey),
3247 epoch: delivery.target_epoch,
3248 },
3249 None,
3250 ),
3251 Err(error) => {
3252 tracing::warn!(%session_id, %error, "durably rejecting one-way supervisor rotation with invalid target");
3253 (
3254 GeneratedSupervisorBinding {
3255 name: raw_target.name,
3256 peer_id: raw_target.peer_id,
3257 address: raw_target.address,
3258 signing_public_key: encode_supervisor_signing_public_key(raw_target.pubkey),
3259 epoch: delivery.target_epoch,
3260 },
3261 Some(
3262 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::InvalidTarget,
3263 ),
3264 )
3265 }
3266 }
3267 } else {
3268 (
3269 GeneratedSupervisorBinding {
3270 name: raw_target.name,
3271 peer_id: raw_target.peer_id,
3272 address: raw_target.address,
3273 signing_public_key: encode_supervisor_signing_public_key(raw_target.pubkey),
3274 epoch: delivery.target_epoch,
3275 },
3276 Some(
3277 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::UnsupportedProtocolVersion,
3278 ),
3279 )
3280 };
3281 if preflight_rejection.is_none()
3282 && let Err(error) = adapter
3283 .stage_local_endpoint_for_comms_runtime(session_id, comms_runtime.as_ref())
3284 .await
3285 {
3286 tracing::warn!(%session_id, %error, "unable to stage local endpoint for supervisor rotation submission");
3287 comms_runtime.mark_interaction_complete(&candidate.interaction.id);
3288 return true;
3289 }
3290 let submission = GeneratedSupervisorRotationSubmit {
3291 operation_id: delivery.operation_id.to_string(),
3292 next,
3293 preflight_rejection,
3294 sender_peer_id,
3295 sender_signing_public_key,
3296 };
3297 match adapter
3298 .submit_supervisor_rotation(session_id, submission)
3299 .await
3300 {
3301 Ok(
3302 SupervisorRotationSubmission::New(_) | SupervisorRotationSubmission::ExistingPending(_),
3303 ) => {
3304 if let Err(error) =
3305 start_supervisor_rotation_worker_if_pending(adapter, session_id, comms_runtime)
3306 .await
3307 {
3308 tracing::warn!(%session_id, %error, "supervisor rotation was durably accepted but worker start failed");
3309 }
3310 }
3311 Ok(SupervisorRotationSubmission::ExistingTerminal) => {}
3312 Ok(
3313 SupervisorRotationSubmission::Rejected(rejection)
3314 | SupervisorRotationSubmission::Conflict(rejection),
3315 ) => {
3316 tracing::warn!(%session_id, ?rejection, "one-way supervisor rotation submission rejected");
3317 }
3318 Err(error) => {
3319 tracing::warn!(%session_id, %error, "one-way supervisor rotation submission failed");
3320 }
3321 }
3322 comms_runtime.mark_interaction_complete(&candidate.interaction.id);
3326 true
3327}
3328
3329async fn serve_hard_cancel_member(
3339 adapter: &Arc<MeerkatMachine>,
3340 session_id: &SessionId,
3341 comms_runtime: &Arc<dyn CommsRuntime>,
3342 sender: &PeerIngressFact,
3343 candidate: &PeerInputCandidate,
3344 payload: &BridgeHardCancelPayload,
3345) {
3346 let sup_payload = BridgeSupervisorPayload {
3351 supervisor: payload.supervisor.clone(),
3352 epoch: payload.epoch,
3353 protocol_version: payload.protocol_version,
3354 };
3355 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
3356 adapter,
3357 session_id,
3358 comms_runtime,
3359 sender,
3360 &sup_payload,
3361 "hard cancel member failed",
3362 )
3363 .await
3364 {
3365 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
3366 return;
3367 }
3368 if let Err((cause, reason)) = require_registered_member_incarnation(
3369 adapter,
3370 session_id,
3371 &payload.expected_member,
3372 "hard cancel member",
3373 ) {
3374 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
3375 return;
3376 }
3377 let deadline = Instant::now() + HARD_CANCEL_SETTLE_TIMEOUT;
3378 loop {
3379 match adapter
3380 .hard_cancel_run_if_current_for_member_incarnation(
3381 session_id,
3382 &payload.expected_run_id,
3383 &payload.expected_member,
3384 payload.reason.clone(),
3385 )
3386 .await
3387 {
3388 Ok(false) => {
3393 if let Err((cause, reason)) = require_registered_member_incarnation(
3394 adapter,
3395 session_id,
3396 &payload.expected_member,
3397 "hard cancel member completion",
3398 ) {
3399 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
3400 return;
3401 }
3402 tracing::debug!(
3403 operation_id = %payload.operation_id,
3404 expected_run_id = %payload.expected_run_id,
3405 "hard-cancel operation reached exact-run terminal truth"
3406 );
3407 send_bridge_response(
3408 comms_runtime,
3409 candidate,
3410 meerkat_core::interaction::ResponseStatus::Completed,
3411 BridgeReply::Ack(BridgeAck { ok: true }),
3412 None,
3413 )
3414 .await;
3415 return;
3416 }
3417 Ok(true) => {}
3418 Err(crate::traits::RuntimeDriverError::StaleAuthority { reason }) => {
3419 send_bridge_failure(
3420 comms_runtime,
3421 candidate,
3422 BridgeRejectionCause::StaleFence,
3423 reason,
3424 None,
3425 )
3426 .await;
3427 return;
3428 }
3429 Err(crate::traits::RuntimeDriverError::NotReady { state }) => {
3432 send_bridge_failure(
3433 comms_runtime,
3434 candidate,
3435 BridgeRejectionCause::Unavailable,
3436 format!(
3437 "hard-cancel operation {} could not cancel expected run {} in state {state:?}",
3438 payload.operation_id, payload.expected_run_id
3439 ),
3440 None,
3441 )
3442 .await;
3443 return;
3444 }
3445 Err(crate::traits::RuntimeDriverError::Destroyed) => {
3446 send_bridge_response(
3451 comms_runtime,
3452 candidate,
3453 meerkat_core::interaction::ResponseStatus::Completed,
3454 BridgeReply::Ack(BridgeAck { ok: true }),
3455 None,
3456 )
3457 .await;
3458 return;
3459 }
3460 Err(error) => {
3461 send_bridge_failure(
3462 comms_runtime,
3463 candidate,
3464 BridgeRejectionCause::Internal,
3465 format!(
3466 "hard-cancel operation {} for expected run {} failed: {error}",
3467 payload.operation_id, payload.expected_run_id
3468 ),
3469 None,
3470 )
3471 .await;
3472 return;
3473 }
3474 }
3475
3476 if Instant::now() >= deadline {
3477 send_bridge_failure(
3478 comms_runtime,
3479 candidate,
3480 BridgeRejectionCause::Unavailable,
3481 format!(
3482 "hard-cancel operation {} timed out waiting for expected run {} to become terminal",
3483 payload.operation_id, payload.expected_run_id
3484 ),
3485 None,
3486 )
3487 .await;
3488 return;
3489 }
3490 crate::tokio::time::sleep(HARD_CANCEL_REASSERT_INTERVAL).await;
3491 }
3492}
3493
3494async fn serve_cancel_tracked_member_input(
3495 adapter: &Arc<MeerkatMachine>,
3496 session_id: &SessionId,
3497 comms_runtime: &Arc<dyn CommsRuntime>,
3498 sender: &PeerIngressFact,
3499 candidate: &PeerInputCandidate,
3500 payload: &BridgeTrackedInputCancelPayload,
3501) {
3502 let sup_payload = BridgeSupervisorPayload {
3503 supervisor: payload.supervisor.clone(),
3504 epoch: payload.epoch,
3505 protocol_version: payload.protocol_version,
3506 };
3507 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
3508 adapter,
3509 session_id,
3510 comms_runtime,
3511 sender,
3512 &sup_payload,
3513 "cancel tracked member input failed",
3514 )
3515 .await
3516 {
3517 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
3518 return;
3519 }
3520 if let Err((cause, reason)) = require_registered_member_incarnation(
3521 adapter,
3522 session_id,
3523 &payload.expected_member,
3524 "cancel tracked member input",
3525 ) {
3526 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
3527 return;
3528 }
3529 match uuid::Uuid::parse_str(&payload.input_id) {
3530 Ok(input_uuid) if !input_uuid.is_nil() && input_uuid.to_string() == payload.input_id => {}
3531 Ok(_) => {
3532 send_bridge_failure(
3533 comms_runtime,
3534 candidate,
3535 BridgeRejectionCause::Unsupported,
3536 "tracked-input cancel input_id must be a canonical non-nil UUID".to_string(),
3537 None,
3538 )
3539 .await;
3540 return;
3541 }
3542 Err(error) => {
3543 send_bridge_failure(
3544 comms_runtime,
3545 candidate,
3546 BridgeRejectionCause::Unsupported,
3547 format!("tracked-input cancel input_id is not a UUID: {error}"),
3548 None,
3549 )
3550 .await;
3551 return;
3552 }
3553 }
3554 let observation = match resolve_member_observation_host(adapter) {
3555 Ok(observation) => observation,
3556 Err((cause, reason)) => {
3557 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
3558 return;
3559 }
3560 };
3561 match observation
3562 .cancel_tracked_member_input(session_id, &payload.expected_member, &payload.input_id)
3563 .await
3564 {
3565 Ok(outcome) => {
3566 send_bridge_response(
3567 comms_runtime,
3568 candidate,
3569 meerkat_core::interaction::ResponseStatus::Completed,
3570 BridgeReply::TrackedInputCancelled(BridgeTrackedInputCancelResponse {
3571 expected_member: payload.expected_member.clone(),
3572 input_id: payload.input_id.clone(),
3573 outcome,
3574 }),
3575 None,
3576 )
3577 .await;
3578 }
3579 Err(error) => {
3580 let (cause, reason) = observation_error_to_bridge_rejection(&error);
3581 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
3582 }
3583 }
3584}
3585
3586async fn try_handle_supervisor_bridge_command(
3587 adapter: &Arc<MeerkatMachine>,
3588 session_id: &SessionId,
3589 comms_runtime: &Arc<dyn CommsRuntime>,
3590 candidate: &PeerInputCandidate,
3591) -> bool {
3592 if try_handle_supervisor_bridge_delivery(adapter, session_id, comms_runtime, candidate).await {
3593 return true;
3594 }
3595 let InteractionContent::Request { intent, params, .. } = &candidate.interaction.content else {
3596 return false;
3597 };
3598
3599 if intent != SUPERVISOR_BRIDGE_INTENT {
3603 return false;
3604 }
3605
3606 if !record_bridge_inbound_peer_request(comms_runtime, candidate) {
3612 return true;
3613 }
3614
3615 let command: BridgeCommand = match decode_bridge_command(params.clone()) {
3616 Ok(cmd) => cmd,
3617 Err(error) => {
3618 let cause = match &error {
3619 meerkat_contracts::wire::supervisor_bridge::BridgeCommandDecodeError::UnsupportedProtocolVersion(_) => {
3620 BridgeRejectionCause::UnsupportedProtocolVersion
3621 }
3622 meerkat_contracts::wire::supervisor_bridge::BridgeCommandDecodeError::Invalid(_) => {
3623 BridgeRejectionCause::Unsupported
3624 }
3625 };
3626 send_bridge_failure(
3627 comms_runtime,
3628 candidate,
3629 cause,
3630 format!("invalid bridge command: {error}"),
3631 None,
3632 )
3633 .await;
3634 return true;
3635 }
3636 };
3637
3638 let sender = &candidate.ingress;
3639
3640 match command {
3641 BridgeCommand::BindMember(payload) => {
3642 let supervisor = match bridge_peer_identity(&payload.supervisor, "bind member failed") {
3643 Ok(supervisor) => supervisor,
3644 Err((cause, reason)) => {
3645 send_bridge_failure(
3646 comms_runtime,
3647 candidate,
3648 cause,
3649 reason,
3650 Some(payload.supervisor.address.as_str()),
3651 )
3652 .await;
3653 return true;
3654 }
3655 };
3656 let admission = match adapter
3657 .resolve_supervisor_bind_admission(
3658 session_id,
3659 supervisor.peer_id.as_str().clone(),
3660 payload.epoch,
3661 generated_sender_peer_id(sender),
3662 )
3663 .await
3664 {
3665 Ok(admission) => admission,
3666 Err(error) => {
3667 send_bridge_failure(
3668 comms_runtime,
3669 candidate,
3670 BridgeRejectionCause::Internal,
3671 format!("bind member admission failed: {error}"),
3672 Some(payload.supervisor.address.as_str()),
3673 )
3674 .await;
3675 return true;
3676 }
3677 };
3678 match admission {
3679 SupervisorBindAdmission::IdempotentAck => {
3680 let Some(advertised) = comms_runtime.advertised_address() else {
3687 send_bridge_failure(
3688 comms_runtime,
3689 candidate,
3690 BridgeRejectionCause::Internal,
3691 "idempotent ack invariant violated",
3692 Some(payload.supervisor.address.as_str()),
3693 )
3694 .await;
3695 return true;
3696 };
3697 let Some(peer_id) = comms_runtime.peer_id().map(|peer_id| peer_id.as_str())
3698 else {
3699 send_bridge_failure(
3700 comms_runtime,
3701 candidate,
3702 BridgeRejectionCause::Internal,
3703 "idempotent ack invariant violated",
3704 Some(payload.supervisor.address.as_str()),
3705 )
3706 .await;
3707 return true;
3708 };
3709 let supervisor_spec =
3710 match TrustedPeerDescriptor::try_from(payload.supervisor.clone()) {
3711 Ok(spec) => spec,
3712 Err(error) => {
3713 send_bridge_failure(
3714 comms_runtime,
3715 candidate,
3716 BridgeRejectionCause::InvalidSupervisorSpec,
3717 format!(
3718 "bind member failed: invalid supervisor peer spec: {error}"
3719 ),
3720 Some(payload.supervisor.address.as_str()),
3721 )
3722 .await;
3723 return true;
3724 }
3725 };
3726 if let Err(error) = republish_private_supervisor_trust(
3727 adapter,
3728 session_id,
3729 comms_runtime.as_ref(),
3730 &supervisor_spec,
3731 payload.epoch,
3732 generated_sender_peer_id(sender),
3733 )
3734 .await
3735 {
3736 send_bridge_failure(
3737 comms_runtime,
3738 candidate,
3739 BridgeRejectionCause::Internal,
3740 format!(
3741 "bind member failed: trust repair failed before idempotent ack: {error}"
3742 ),
3743 Some(payload.supervisor.address.as_str()),
3744 )
3745 .await;
3746 return true;
3747 }
3748 send_bridge_response(
3749 comms_runtime,
3750 candidate,
3751 meerkat_core::interaction::ResponseStatus::Completed,
3752 BridgeReply::BindMember(BridgeBindResponse {
3753 peer_id,
3754 address: canonicalize_bridge_address(&advertised),
3755 capabilities: bridge_capabilities(payload.protocol_version),
3756 }),
3757 None,
3758 )
3759 .await;
3760 return true;
3761 }
3762 SupervisorBindAdmission::Bootstrap => {}
3763 SupervisorBindAdmission::Rejected(rejection) => {
3764 send_bridge_failure(
3765 comms_runtime,
3766 candidate,
3767 supervisor_bind_admission_rejection_cause(rejection),
3768 supervisor_bind_admission_rejection_reason(rejection, sender, &supervisor),
3769 Some(payload.supervisor.address.as_str()),
3770 )
3771 .await;
3772 return true;
3773 }
3774 }
3775 let (supervisor_spec, advertised_address) =
3776 match validate_bind_request(adapter, session_id, comms_runtime, sender, &payload)
3777 .await
3778 {
3779 Ok(binding) => binding,
3780 Err((cause, reason)) => {
3781 send_bridge_failure(
3782 comms_runtime,
3783 candidate,
3784 cause,
3785 reason,
3786 Some(payload.supervisor.address.as_str()),
3787 )
3788 .await;
3789 return true;
3790 }
3791 };
3792 let Some(peer_id) = comms_runtime.peer_id().map(|peer_id| peer_id.as_str()) else {
3799 send_bridge_failure(
3800 comms_runtime,
3801 candidate,
3802 BridgeRejectionCause::Internal,
3803 "bind member failed: runtime peer_id unavailable",
3804 Some(payload.supervisor.address.as_str()),
3805 )
3806 .await;
3807 return true;
3808 };
3809 if let Err(reason) = commit_supervisor_bind_with_rollback(
3817 adapter,
3818 session_id,
3819 comms_runtime.as_ref(),
3820 &supervisor_spec,
3821 payload.epoch,
3822 "bind member",
3823 )
3824 .await
3825 {
3826 send_bridge_failure(
3827 comms_runtime,
3828 candidate,
3829 BridgeRejectionCause::Internal,
3830 reason,
3831 Some(payload.supervisor.address.as_str()),
3832 )
3833 .await;
3834 return true;
3835 }
3836 send_bridge_response(
3837 comms_runtime,
3838 candidate,
3839 meerkat_core::interaction::ResponseStatus::Completed,
3840 BridgeReply::BindMember(BridgeBindResponse {
3841 peer_id,
3842 address: canonicalize_bridge_address(&advertised_address),
3843 capabilities: bridge_capabilities(payload.protocol_version),
3844 }),
3845 None,
3846 )
3847 .await;
3848 true
3849 }
3850 BridgeCommand::AuthorizeSupervisor(payload) => {
3851 handle_authorize_supervisor_command(
3852 adapter,
3853 session_id,
3854 comms_runtime,
3855 candidate,
3856 sender,
3857 payload,
3858 )
3859 .await
3860 }
3861 BridgeCommand::RevokeSupervisor(payload) => {
3862 let authorized_supervisor = match resolve_authorized_supervisor_cleanup(
3863 adapter,
3864 session_id,
3865 sender,
3866 &payload,
3867 crate::meerkat_machine::dsl::SupervisorCleanupCommandKind::Revoke,
3868 )
3869 .await
3870 {
3871 Ok(supervisor) => supervisor,
3872 Err((BridgeRejectionCause::NotBound, reason)) => {
3873 send_bridge_failure(
3874 comms_runtime,
3875 candidate,
3876 BridgeRejectionCause::NotBound,
3877 reason,
3878 Some(payload.supervisor.address.as_str()),
3879 )
3880 .await;
3881 return true;
3882 }
3883 Err((cause, reason)) => {
3884 send_bridge_failure(
3885 comms_runtime,
3886 candidate,
3887 cause,
3888 reason,
3889 Some(payload.supervisor.address.as_str()),
3890 )
3891 .await;
3892 return true;
3893 }
3894 };
3895 let resume_pending_revoke = authorized_supervisor.resume_pending_revoke;
3896 let authorized_supervisor = authorized_supervisor.supervisor;
3897 if let Err(error) = adapter
3898 .stage_local_endpoint_for_comms_runtime(session_id, comms_runtime.as_ref())
3899 .await
3900 {
3901 send_bridge_failure(
3902 comms_runtime,
3903 candidate,
3904 BridgeRejectionCause::Internal,
3905 format!("revoke supervisor failed: local endpoint rejected: {error}"),
3906 Some(payload.supervisor.address.as_str()),
3907 )
3908 .await;
3909 return true;
3910 }
3911 let supervisor_peer_id = authorized_supervisor.peer_id.as_str();
3912 let revoke_transition = match adapter
3913 .stage_supervisor_revoke(session_id, supervisor_peer_id.clone(), payload.epoch)
3914 .await
3915 {
3916 Ok(transition) => transition,
3917 Err(error) => {
3918 send_bridge_failure(
3919 comms_runtime,
3920 candidate,
3921 BridgeRejectionCause::Internal,
3922 format!("revoke supervisor failed: DSL rejected revoke: {error}"),
3923 Some(payload.supervisor.address.as_str()),
3924 )
3925 .await;
3926 return true;
3927 }
3928 };
3929 let revoke_obligation = match matching_supervisor_revoke_obligation(
3930 adapter,
3931 session_id,
3932 &revoke_transition,
3933 supervisor_peer_id.as_str(),
3934 payload.epoch,
3935 "revoke supervisor",
3936 )
3937 .await
3938 {
3939 Ok(obligation) => obligation,
3940 Err(reason) => {
3941 if let Err(error) = adapter
3942 .stage_supervisor_trust_revoke_failed(
3943 session_id,
3944 supervisor_peer_id.clone(),
3945 payload.epoch,
3946 reason.clone(),
3947 )
3948 .await
3949 {
3950 tracing::warn!(
3951 %session_id,
3952 epoch = payload.epoch,
3953 %error,
3954 "failed to restore supervisor binding after missing revoke effect"
3955 );
3956 }
3957 send_bridge_failure(
3958 comms_runtime,
3959 candidate,
3960 BridgeRejectionCause::Internal,
3961 reason,
3962 Some(payload.supervisor.address.as_str()),
3963 )
3964 .await;
3965 return true;
3966 }
3967 };
3968 let revoke_authority = match supervisor_revoke_authority(&revoke_obligation) {
3969 Ok(authority) => authority,
3970 Err(error) => {
3971 let _ = adapter
3972 .stage_supervisor_trust_revoke_failed(
3973 session_id,
3974 revoke_obligation.peer_id().clone(),
3975 revoke_obligation.epoch(),
3976 error.clone(),
3977 )
3978 .await;
3979 send_bridge_failure(
3980 comms_runtime,
3981 candidate,
3982 BridgeRejectionCause::Internal,
3983 error,
3984 Some(payload.supervisor.address.as_str()),
3985 )
3986 .await;
3987 return true;
3988 }
3989 };
3990 if resume_pending_revoke {
3995 send_bridge_failure(
3996 comms_runtime,
3997 candidate,
3998 BridgeRejectionCause::NotBound,
3999 "supervisor revoke cleanup was already pending".to_string(),
4000 Some(payload.supervisor.address.as_str()),
4001 )
4002 .await;
4003 } else {
4004 send_bridge_response(
4005 comms_runtime,
4006 candidate,
4007 meerkat_core::interaction::ResponseStatus::Completed,
4008 BridgeReply::Ack(BridgeAck { ok: true }),
4009 Some(payload.supervisor.address.as_str()),
4010 )
4011 .await;
4012 }
4013 if let Err(error) = apply_generated_private_trust_remove(
4014 comms_runtime.as_ref(),
4015 revoke_obligation.peer_id().clone(),
4016 revoke_authority,
4017 )
4018 .await
4019 {
4020 let feedback_result = adapter
4021 .stage_supervisor_trust_revoke_failed(
4022 session_id,
4023 revoke_obligation.peer_id().clone(),
4024 revoke_obligation.epoch(),
4025 error.clone(),
4026 )
4027 .await;
4028 let mut reason = format!(
4029 "revoke supervisor failed: trust removal rejected after DSL commit: {error}"
4030 );
4031 if let Err(feedback_error) = feedback_result {
4032 reason.push_str(&format!("; feedback failed: {feedback_error}"));
4033 }
4034 tracing::warn!(%session_id, %reason, "accepted supervisor revoke remains pending");
4035 return true;
4036 }
4037 if let Err(error) = adapter
4043 .stage_supervisor_trust_revoked(
4044 session_id,
4045 revoke_obligation.peer_id().clone(),
4046 revoke_obligation.epoch(),
4047 )
4048 .await
4049 {
4050 tracing::warn!(%session_id, %error, "accepted supervisor revoke cleanup checkpoint failed");
4051 return true;
4052 }
4053 true
4054 }
4055 BridgeCommand::DeliverMemberInput(payload) => {
4056 let sup_payload = BridgeSupervisorPayload {
4057 supervisor: payload.supervisor.clone(),
4058 epoch: payload.epoch,
4059 protocol_version: payload.protocol_version,
4060 };
4061 let authorized_supervisor = match resolve_authorized_supervisor_with_response_route(
4062 adapter,
4063 session_id,
4064 comms_runtime,
4065 sender,
4066 &sup_payload,
4067 "deliver member input failed",
4068 )
4069 .await
4070 {
4071 Ok(supervisor) => supervisor,
4072 Err((cause, reason)) => {
4073 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4074 return true;
4075 }
4076 };
4077 let request_input_id = payload.input_id.clone();
4078 let registered_incarnation = adapter.member_incarnation(session_id);
4079 match (®istered_incarnation, &payload.expected_member) {
4080 (Some(current), Some(expected)) if current == expected => {}
4081 (Some(current), _) => {
4082 let response = BridgeDeliveryResponse {
4083 input_id: request_input_id,
4084 canonical_input_id: None,
4085 outcome: BridgeDeliveryOutcome::Rejected {
4086 cause: BridgeDeliveryRejectionCause::StaleMemberIncarnation {
4087 current: current.clone(),
4088 },
4089 reason: format!(
4090 "delivery expected member incarnation {:?}; current is {:?}",
4091 payload.expected_member, current
4092 ),
4093 },
4094 };
4095 send_bridge_response(
4096 comms_runtime,
4097 candidate,
4098 meerkat_core::interaction::ResponseStatus::Completed,
4099 BridgeReply::Delivery(response),
4100 None,
4101 )
4102 .await;
4103 return true;
4104 }
4105 (None, Some(expected)) => {
4106 let response = BridgeDeliveryResponse {
4107 input_id: request_input_id,
4108 canonical_input_id: None,
4109 outcome: BridgeDeliveryOutcome::Rejected {
4110 cause: BridgeDeliveryRejectionCause::Internal {
4111 detail: "target session has no registered host-member incarnation"
4112 .to_string(),
4113 },
4114 reason: format!(
4115 "delivery addressed host member {expected:?}, but target session has no host journal"
4116 ),
4117 },
4118 };
4119 send_bridge_response(
4120 comms_runtime,
4121 candidate,
4122 meerkat_core::interaction::ResponseStatus::Completed,
4123 BridgeReply::Delivery(response),
4124 None,
4125 )
4126 .await;
4127 return true;
4128 }
4129 (None, None) => {}
4130 }
4131 if let Err(detail) = validate_delivery_tracking_request(&payload) {
4137 let response = BridgeDeliveryResponse {
4138 input_id: request_input_id,
4139 canonical_input_id: None,
4140 outcome: BridgeDeliveryOutcome::Rejected {
4141 cause: BridgeDeliveryRejectionCause::TurnDirectiveUnsupported {
4142 detail: detail.clone(),
4143 },
4144 reason: detail,
4145 },
4146 };
4147 send_bridge_response(
4148 comms_runtime,
4149 candidate,
4150 meerkat_core::interaction::ResponseStatus::Completed,
4151 BridgeReply::Delivery(response),
4152 None,
4153 )
4154 .await;
4155 return true;
4156 }
4157 if delivery_requires_tracked_turn_custody(&payload) {
4163 serve_tracked_member_delivery(
4164 adapter,
4165 session_id,
4166 comms_runtime,
4167 candidate,
4168 authorized_supervisor.peer_id,
4169 payload,
4170 )
4171 .await;
4172 return true;
4173 }
4174 let expected_member = payload.expected_member.clone();
4175 let input = match peer_input_from_delivery_payload(
4176 session_id,
4177 authorized_supervisor.peer_id,
4178 payload,
4179 ) {
4180 Ok(input) => input,
4181 Err(reason) => {
4182 let response = BridgeDeliveryResponse {
4183 input_id: request_input_id,
4184 canonical_input_id: None,
4185 outcome: BridgeDeliveryOutcome::Rejected {
4186 cause: BridgeDeliveryRejectionCause::Internal {
4187 detail: reason.clone(),
4188 },
4189 reason,
4190 },
4191 };
4192 send_bridge_response(
4193 comms_runtime,
4194 candidate,
4195 meerkat_core::interaction::ResponseStatus::Completed,
4196 BridgeReply::Delivery(response),
4197 None,
4198 )
4199 .await;
4200 return true;
4201 }
4202 };
4203 let accept_result = adapter
4204 .accept_input_with_completion_for_member_residency(
4205 session_id,
4206 input,
4207 expected_member.as_ref(),
4208 )
4209 .await;
4210 match accept_result {
4211 Ok((accept_outcome, _completion_handle)) => {
4212 let response = match accept_outcome {
4213 crate::accept::AcceptOutcome::Accepted { input_id, .. } => {
4214 BridgeDeliveryResponse {
4215 input_id: request_input_id,
4216 canonical_input_id: Some(input_id.to_string()),
4217 outcome: BridgeDeliveryOutcome::Accepted,
4218 }
4219 }
4220 crate::accept::AcceptOutcome::Deduplicated { existing_id, .. } => {
4221 let existing_id = existing_id.to_string();
4222 BridgeDeliveryResponse {
4223 input_id: request_input_id,
4224 canonical_input_id: Some(existing_id.clone()),
4225 outcome: BridgeDeliveryOutcome::Deduplicated {
4226 existing_input_id: existing_id,
4227 },
4228 }
4229 }
4230 crate::accept::AcceptOutcome::Rejected { reason } => {
4231 let cause = bridge_delivery_rejection_cause(&reason);
4232 BridgeDeliveryResponse {
4233 input_id: request_input_id,
4234 canonical_input_id: None,
4235 outcome: BridgeDeliveryOutcome::Rejected {
4236 cause,
4237 reason: reason.to_string(),
4238 },
4239 }
4240 }
4241 };
4242 send_bridge_response(
4243 comms_runtime,
4244 candidate,
4245 meerkat_core::interaction::ResponseStatus::Completed,
4246 BridgeReply::Delivery(response),
4247 None,
4248 )
4249 .await;
4250 }
4251 Err(crate::traits::RuntimeDriverError::StaleAuthority { reason }) => {
4252 if let Some(current) = adapter.member_incarnation(session_id) {
4253 let response = BridgeDeliveryResponse {
4254 input_id: request_input_id,
4255 canonical_input_id: None,
4256 outcome: BridgeDeliveryOutcome::Rejected {
4257 cause: BridgeDeliveryRejectionCause::StaleMemberIncarnation {
4258 current,
4259 },
4260 reason,
4261 },
4262 };
4263 send_bridge_response(
4264 comms_runtime,
4265 candidate,
4266 meerkat_core::interaction::ResponseStatus::Completed,
4267 BridgeReply::Delivery(response),
4268 None,
4269 )
4270 .await;
4271 } else {
4272 send_bridge_failure(
4273 comms_runtime,
4274 candidate,
4275 BridgeRejectionCause::StaleFence,
4276 reason,
4277 None,
4278 )
4279 .await;
4280 }
4281 }
4282 Err(error) => {
4283 send_bridge_failure(
4284 comms_runtime,
4285 candidate,
4286 BridgeRejectionCause::Internal,
4287 format!("deliver member input failed: {error}"),
4288 None,
4289 )
4290 .await;
4291 }
4292 }
4293 true
4294 }
4295 BridgeCommand::InterruptMember(payload) => {
4296 let sup_payload = BridgeSupervisorPayload {
4297 supervisor: payload.supervisor.clone(),
4298 epoch: payload.epoch,
4299 protocol_version: payload.protocol_version,
4300 };
4301 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
4302 adapter,
4303 session_id,
4304 comms_runtime,
4305 sender,
4306 &sup_payload,
4307 "interrupt member failed",
4308 )
4309 .await
4310 {
4311 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4312 return true;
4313 }
4314 if let Err((cause, reason)) = require_optional_registered_member_incarnation(
4315 adapter,
4316 session_id,
4317 payload.expected_member.as_ref(),
4318 "interrupt member",
4319 ) {
4320 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4321 return true;
4322 }
4323 let result = adapter
4324 .cancel_after_boundary_for_optional_member_incarnation(
4325 session_id,
4326 payload.expected_member.as_ref(),
4327 )
4328 .await;
4329 match result {
4330 Ok(()) => {
4331 if let Err((cause, reason)) = require_optional_registered_member_incarnation(
4332 adapter,
4333 session_id,
4334 payload.expected_member.as_ref(),
4335 "interrupt member completion",
4336 ) {
4337 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4338 return true;
4339 }
4340 send_bridge_response(
4341 comms_runtime,
4342 candidate,
4343 meerkat_core::interaction::ResponseStatus::Completed,
4344 BridgeReply::Ack(BridgeAck { ok: true }),
4345 None,
4346 )
4347 .await;
4348 }
4349 Err(crate::traits::RuntimeDriverError::StaleAuthority { reason }) => {
4350 send_bridge_failure(
4351 comms_runtime,
4352 candidate,
4353 BridgeRejectionCause::StaleFence,
4354 reason,
4355 None,
4356 )
4357 .await;
4358 }
4359 Err(error) => {
4360 send_bridge_failure(
4361 comms_runtime,
4362 candidate,
4363 BridgeRejectionCause::Internal,
4364 format!("interrupt member failed: {error}"),
4365 None,
4366 )
4367 .await;
4368 }
4369 }
4370 true
4371 }
4372 BridgeCommand::RetireMember(payload) => {
4373 if let Err((cause, reason)) = resolve_authorized_supervisor_cleanup_with_response_route(
4374 adapter,
4375 session_id,
4376 comms_runtime,
4377 sender,
4378 &payload,
4379 crate::meerkat_machine::dsl::SupervisorCleanupCommandKind::Retire,
4380 "retire member failed",
4381 )
4382 .await
4383 {
4384 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4385 return true;
4386 }
4387 match adapter.retire_runtime(session_id).await {
4388 Ok(report) => {
4389 send_bridge_response(
4390 comms_runtime,
4391 candidate,
4392 meerkat_core::interaction::ResponseStatus::Completed,
4393 BridgeReply::Retire(BridgeRetireResponse {
4394 inputs_abandoned: report.inputs_abandoned,
4395 inputs_pending_drain: report.inputs_pending_drain,
4396 }),
4397 None,
4398 )
4399 .await;
4400 }
4401 Err(error) => {
4402 send_bridge_failure(
4403 comms_runtime,
4404 candidate,
4405 BridgeRejectionCause::Internal,
4406 format!("retire member failed: {error}"),
4407 None,
4408 )
4409 .await;
4410 }
4411 }
4412 true
4413 }
4414 BridgeCommand::DestroyMember(payload) => {
4415 if let Err((cause, reason)) = resolve_authorized_supervisor_cleanup_with_response_route(
4416 adapter,
4417 session_id,
4418 comms_runtime,
4419 sender,
4420 &payload,
4421 crate::meerkat_machine::dsl::SupervisorCleanupCommandKind::Destroy,
4422 "destroy member failed",
4423 )
4424 .await
4425 {
4426 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4427 return true;
4428 }
4429 let runtime_id = MeerkatMachine::logical_runtime_id(session_id);
4430 match RuntimeControlPlane::destroy(adapter.as_ref(), &runtime_id).await {
4431 Ok(report) => {
4432 send_bridge_response(
4433 comms_runtime,
4434 candidate,
4435 meerkat_core::interaction::ResponseStatus::Completed,
4436 BridgeReply::Destroy(BridgeDestroyResponse {
4437 inputs_abandoned: report.inputs_abandoned,
4438 }),
4439 None,
4440 )
4441 .await;
4442 }
4443 Err(error) => {
4444 send_bridge_failure(
4445 comms_runtime,
4446 candidate,
4447 BridgeRejectionCause::Internal,
4448 format!("destroy member failed: {error}"),
4449 None,
4450 )
4451 .await;
4452 }
4453 }
4454 true
4455 }
4456 BridgeCommand::ObserveMember(payload) => {
4457 if let Err((cause, reason)) = resolve_authorized_supervisor_cleanup_with_response_route(
4458 adapter,
4459 session_id,
4460 comms_runtime,
4461 sender,
4462 &payload,
4463 crate::meerkat_machine::dsl::SupervisorCleanupCommandKind::Observe,
4464 "observe member failed",
4465 )
4466 .await
4467 {
4468 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4469 return true;
4470 }
4471 match crate::service_ext::SessionServiceRuntimeExt::runtime_state(
4472 adapter.as_ref(),
4473 session_id,
4474 )
4475 .await
4476 {
4477 Ok(state) => {
4478 let current_run_id = adapter
4479 .meerkat_machine_spine_snapshot(session_id)
4480 .await
4481 .and_then(|snapshot| {
4482 snapshot
4483 .control
4484 .current_run_id
4485 .map(|run_id| run_id.to_string())
4486 });
4487 let bridge_state = runtime_state_to_bridge(state);
4488 let lifecycle_facts =
4489 match crate::meerkat_machine::classify_runtime_lifecycle_state(state) {
4490 Ok(facts) => facts,
4491 Err(error) => {
4492 send_bridge_failure(
4493 comms_runtime,
4494 candidate,
4495 BridgeRejectionCause::Internal,
4496 format!("runtime lifecycle classification failed: {error}"),
4497 None,
4498 )
4499 .await;
4500 return true;
4501 }
4502 };
4503 send_bridge_response(
4504 comms_runtime,
4505 candidate,
4506 meerkat_core::interaction::ResponseStatus::Completed,
4507 BridgeReply::Observation(BridgeObservationResponse::new(
4508 bridge_state,
4509 Some(lifecycle_facts.can_accept_input()),
4510 current_run_id,
4511 Some(BridgePeerConnectivity::Reachable),
4512 None,
4513 chrono::Utc::now().to_rfc3339(),
4514 )),
4515 None,
4516 )
4517 .await;
4518 }
4519 Err(error) => {
4520 send_bridge_failure(
4521 comms_runtime,
4522 candidate,
4523 BridgeRejectionCause::Internal,
4524 format!("observe member failed: {error}"),
4525 None,
4526 )
4527 .await;
4528 }
4529 }
4530 true
4531 }
4532 BridgeCommand::WireMember(payload) => {
4533 let sup_payload = BridgeSupervisorPayload {
4534 supervisor: payload.supervisor.clone(),
4535 epoch: payload.epoch,
4536 protocol_version: payload.protocol_version,
4537 };
4538 let authorized_supervisor = match resolve_authorized_supervisor_with_response_route(
4539 adapter,
4540 session_id,
4541 comms_runtime,
4542 sender,
4543 &sup_payload,
4544 "wire member failed",
4545 )
4546 .await
4547 {
4548 Ok(supervisor) => supervisor,
4549 Err((cause, reason)) => {
4550 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4551 return true;
4552 }
4553 };
4554 let peer_spec = match TrustedPeerDescriptor::try_from(payload.peer_spec) {
4562 Ok(spec) => spec,
4563 Err(error) => {
4564 send_bridge_failure(
4565 comms_runtime,
4566 candidate,
4567 BridgeRejectionCause::InvalidPeerSpec,
4568 format!("wire member failed: invalid trusted peer spec: {error}"),
4569 None,
4570 )
4571 .await;
4572 return true;
4573 }
4574 };
4575 let mob_peer_overlay = match payload.mob_peer_overlay {
4580 Some(overlay) => overlay,
4581 None => {
4582 send_bridge_failure(
4583 comms_runtime,
4584 candidate,
4585 BridgeRejectionCause::UnsupportedProtocolVersion,
4586 format!(
4587 "wire member failed: peer wiring requires supervisor bridge protocol v{} (mob peer overlay absent); upgrade the supervisor",
4588 BridgeProtocolVersion::V3
4589 ),
4590 None,
4591 )
4592 .await;
4593 return true;
4594 }
4595 };
4596 let overlay = match bridge_mob_peer_overlay(&mob_peer_overlay) {
4597 Ok(overlay) => overlay,
4598 Err(error) => {
4599 send_bridge_failure(
4600 comms_runtime,
4601 candidate,
4602 BridgeRejectionCause::InvalidPeerSpec,
4603 format!("wire member failed: invalid MobMachine peer overlay: {error}"),
4604 None,
4605 )
4606 .await;
4607 return true;
4608 }
4609 };
4610 match adapter
4611 .stage_authorized_supervisor_mob_peer_overlay(
4612 session_id,
4613 payload.supervisor.peer_id,
4614 payload.epoch,
4615 mob_peer_overlay.recipient_peer_id,
4616 mob_peer_overlay.topology_epoch,
4617 overlay.endpoints,
4618 overlay.endpoint_count,
4619 peer_spec.peer_id.to_string(),
4620 crate::meerkat_machine::dsl::PeerEndpoint::from(&peer_spec),
4621 crate::meerkat_machine::dsl::MobPeerOverlayCommandKind::Wire,
4622 Arc::clone(comms_runtime),
4623 )
4624 .await
4625 {
4626 Ok(()) => {
4627 if let Err((cause, reason)) = ensure_bridge_response_route_for_supervisor(
4628 adapter,
4629 session_id,
4630 comms_runtime,
4631 &authorized_supervisor,
4632 "wire member failed",
4633 )
4634 .await
4635 {
4636 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4637 return true;
4638 }
4639 send_bridge_response(
4640 comms_runtime,
4641 candidate,
4642 meerkat_core::interaction::ResponseStatus::Completed,
4643 BridgeReply::Ack(BridgeAck { ok: true }),
4644 None,
4645 )
4646 .await;
4647 }
4648 Err(error) => {
4649 let endpoint = crate::meerkat_machine::dsl::PeerEndpoint::from(&peer_spec);
4656 if matches!(
4657 error,
4658 crate::meerkat_machine::PeerEndpointStageError::Dsl(
4659 crate::meerkat_machine::dsl::MeerkatMachineTransitionError::GuardRejected {
4660 ..
4661 }
4662 )
4663 ) && adapter
4664 .direct_peer_endpoints(session_id)
4665 .await
4666 .map(|endpoints| endpoints.contains(&endpoint))
4667 .unwrap_or(false)
4668 {
4669 if let Err((cause, reason)) = ensure_bridge_response_route_for_supervisor(
4670 adapter,
4671 session_id,
4672 comms_runtime,
4673 &authorized_supervisor,
4674 "wire member failed",
4675 )
4676 .await
4677 {
4678 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4679 return true;
4680 }
4681 send_bridge_response(
4682 comms_runtime,
4683 candidate,
4684 meerkat_core::interaction::ResponseStatus::Completed,
4685 BridgeReply::Ack(BridgeAck { ok: true }),
4686 None,
4687 )
4688 .await;
4689 } else {
4690 send_bridge_failure(
4691 comms_runtime,
4692 candidate,
4693 BridgeRejectionCause::Internal,
4694 format!(
4695 "wire member failed: generated peer overlay authority rejected: {error}"
4696 ),
4697 None,
4698 )
4699 .await;
4700 }
4701 }
4702 }
4703 true
4704 }
4705 BridgeCommand::UnwireMember(payload) => {
4706 let sup_payload = BridgeSupervisorPayload {
4707 supervisor: payload.supervisor.clone(),
4708 epoch: payload.epoch,
4709 protocol_version: payload.protocol_version,
4710 };
4711 let authorized_supervisor = match resolve_authorized_supervisor_with_response_route(
4712 adapter,
4713 session_id,
4714 comms_runtime,
4715 sender,
4716 &sup_payload,
4717 "unwire member failed",
4718 )
4719 .await
4720 {
4721 Ok(supervisor) => supervisor,
4722 Err((cause, reason)) => {
4723 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4724 return true;
4725 }
4726 };
4727 let peer_spec = match TrustedPeerDescriptor::try_from(payload.peer_spec) {
4734 Ok(spec) => spec,
4735 Err(error) => {
4736 send_bridge_failure(
4737 comms_runtime,
4738 candidate,
4739 BridgeRejectionCause::InvalidPeerSpec,
4740 format!("unwire member failed: invalid trusted peer spec: {error}"),
4741 None,
4742 )
4743 .await;
4744 return true;
4745 }
4746 };
4747 let mob_peer_overlay = match payload.mob_peer_overlay {
4750 Some(overlay) => overlay,
4751 None => {
4752 send_bridge_failure(
4753 comms_runtime,
4754 candidate,
4755 BridgeRejectionCause::UnsupportedProtocolVersion,
4756 format!(
4757 "unwire member failed: peer wiring requires supervisor bridge protocol v{} (mob peer overlay absent); upgrade the supervisor",
4758 BridgeProtocolVersion::V3
4759 ),
4760 None,
4761 )
4762 .await;
4763 return true;
4764 }
4765 };
4766 let overlay = match bridge_mob_peer_overlay(&mob_peer_overlay) {
4767 Ok(overlay) => overlay,
4768 Err(error) => {
4769 send_bridge_failure(
4770 comms_runtime,
4771 candidate,
4772 BridgeRejectionCause::InvalidPeerSpec,
4773 format!("unwire member failed: invalid MobMachine peer overlay: {error}"),
4774 None,
4775 )
4776 .await;
4777 return true;
4778 }
4779 };
4780 match adapter
4781 .stage_authorized_supervisor_mob_peer_overlay(
4782 session_id,
4783 payload.supervisor.peer_id,
4784 payload.epoch,
4785 mob_peer_overlay.recipient_peer_id,
4786 mob_peer_overlay.topology_epoch,
4787 overlay.endpoints,
4788 overlay.endpoint_count,
4789 peer_spec.peer_id.to_string(),
4790 crate::meerkat_machine::dsl::PeerEndpoint::from(&peer_spec),
4791 crate::meerkat_machine::dsl::MobPeerOverlayCommandKind::Unwire,
4792 Arc::clone(comms_runtime),
4793 )
4794 .await
4795 {
4796 Ok(()) => {
4797 if let Err((cause, reason)) = ensure_bridge_response_route_for_supervisor(
4798 adapter,
4799 session_id,
4800 comms_runtime,
4801 &authorized_supervisor,
4802 "unwire member failed",
4803 )
4804 .await
4805 {
4806 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4807 return true;
4808 }
4809 send_bridge_response(
4810 comms_runtime,
4811 candidate,
4812 meerkat_core::interaction::ResponseStatus::Completed,
4813 BridgeReply::Ack(BridgeAck { ok: true }),
4814 None,
4815 )
4816 .await;
4817 }
4818 Err(error) => {
4819 let endpoint = crate::meerkat_machine::dsl::PeerEndpoint::from(&peer_spec);
4825 if matches!(
4826 error,
4827 crate::meerkat_machine::PeerEndpointStageError::Dsl(
4828 crate::meerkat_machine::dsl::MeerkatMachineTransitionError::GuardRejected {
4829 ..
4830 }
4831 )
4832 ) && !adapter
4833 .direct_peer_endpoints(session_id)
4834 .await
4835 .map(|endpoints| endpoints.contains(&endpoint))
4836 .unwrap_or(false)
4837 {
4838 if let Err((cause, reason)) = ensure_bridge_response_route_for_supervisor(
4839 adapter,
4840 session_id,
4841 comms_runtime,
4842 &authorized_supervisor,
4843 "unwire member failed",
4844 )
4845 .await
4846 {
4847 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
4848 return true;
4849 }
4850 send_bridge_response(
4851 comms_runtime,
4852 candidate,
4853 meerkat_core::interaction::ResponseStatus::Completed,
4854 BridgeReply::Ack(BridgeAck { ok: true }),
4855 None,
4856 )
4857 .await;
4858 } else {
4859 send_bridge_failure(
4860 comms_runtime,
4861 candidate,
4862 BridgeRejectionCause::Internal,
4863 format!(
4864 "unwire member failed: generated peer overlay authority rejected: {error}"
4865 ),
4866 None,
4867 )
4868 .await;
4869 }
4870 }
4871 }
4872 true
4873 }
4874 BridgeCommand::ObserveSupervisorRotation(payload) => {
4875 if payload.protocol_version != BridgeProtocolVersion::V4 {
4876 send_bridge_failure(
4877 comms_runtime,
4878 candidate,
4879 BridgeRejectionCause::UnsupportedProtocolVersion,
4880 format!(
4881 "observe supervisor rotation requires protocol v4, received {:?}",
4882 payload.protocol_version
4883 ),
4884 Some(payload.observer.address.as_str()),
4885 )
4886 .await;
4887 return true;
4888 }
4889 let observer =
4890 match bridge_peer_identity(&payload.observer, "observe supervisor rotation failed")
4891 {
4892 Ok(observer) => observer,
4893 Err((cause, reason)) => {
4894 send_bridge_failure(
4895 comms_runtime,
4896 candidate,
4897 cause,
4898 reason,
4899 Some(payload.observer.address.as_str()),
4900 )
4901 .await;
4902 return true;
4903 }
4904 };
4905 let authenticated_peer_id = generated_sender_peer_id(sender);
4906 let authenticated_key = generated_sender_signing_public_key(sender);
4907 let observer_peer_id = observer.peer_id.as_str();
4908 let observer_key = encode_supervisor_signing_public_key(observer.pubkey.into_bytes());
4909 if authenticated_peer_id.as_deref() != Some(observer_peer_id.as_str())
4910 || authenticated_key.as_deref() != Some(observer_key.as_str())
4911 {
4912 send_bridge_failure(
4913 comms_runtime,
4914 candidate,
4915 BridgeRejectionCause::SenderMismatch,
4916 "observe supervisor rotation failed: observer does not match authenticated sender"
4917 .to_string(),
4918 Some(payload.observer.address.as_str()),
4919 )
4920 .await;
4921 return true;
4922 }
4923 let operation_id = payload.operation_id;
4924 let observer = GeneratedSupervisorBinding {
4925 name: observer.name.as_str().to_owned(),
4926 peer_id: observer_peer_id,
4927 address: observer.address.to_string(),
4928 signing_public_key: observer_key,
4929 epoch: payload.observer_epoch,
4930 };
4931 let reply =
4932 match adapter
4933 .observe_supervisor_rotation(session_id, operation_id.to_string(), &observer)
4934 .await
4935 {
4936 Ok(SupervisorRotationObservation::Found(receipt)) => {
4937 match bridge_supervisor_rotation_state(receipt) {
4938 Ok(state) => BridgeReply::SupervisorRotation(
4939 BridgeSupervisorRotationObservation::Found { state },
4940 ),
4941 Err(error) => {
4942 send_bridge_failure(
4943 comms_runtime,
4944 candidate,
4945 BridgeRejectionCause::Internal,
4946 error,
4947 Some(payload.observer.address.as_str()),
4948 )
4949 .await;
4950 return true;
4951 }
4952 }
4953 }
4954 Ok(SupervisorRotationObservation::NotFound) => BridgeReply::SupervisorRotation(
4955 BridgeSupervisorRotationObservation::NotFound { operation_id },
4956 ),
4957 Err(error) => {
4958 send_bridge_failure(
4959 comms_runtime,
4960 candidate,
4961 BridgeRejectionCause::Internal,
4962 format!("observe supervisor rotation failed: {error}"),
4963 Some(payload.observer.address.as_str()),
4964 )
4965 .await;
4966 return true;
4967 }
4968 };
4969 send_bridge_response(
4977 comms_runtime,
4978 candidate,
4979 meerkat_core::interaction::ResponseStatus::Completed,
4980 reply,
4981 Some(payload.observer.address.as_str()),
4982 )
4983 .await;
4984 true
4985 }
4986 BridgeCommand::DeclareMemberOutboundTaint(payload) => {
4987 let sup_payload = BridgeSupervisorPayload {
4988 supervisor: payload.supervisor.clone(),
4989 epoch: payload.epoch,
4990 protocol_version: payload.protocol_version,
4991 };
4992 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
4993 adapter,
4994 session_id,
4995 comms_runtime,
4996 sender,
4997 &sup_payload,
4998 "declare member outbound taint failed",
4999 )
5000 .await
5001 {
5002 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5003 return true;
5004 }
5005 let incarnation_check = match payload.target.as_ref() {
5006 Some(BridgeOutboundTaintTarget::Placed(expected)) => {
5007 require_registered_member_incarnation(
5008 adapter,
5009 session_id,
5010 expected,
5011 "declare member outbound taint failed",
5012 )
5013 }
5014 Some(BridgeOutboundTaintTarget::PeerOnly) | None => {
5015 if let Some(current) = adapter.member_incarnation(session_id) {
5016 Err((
5017 BridgeRejectionCause::StaleFence,
5018 format!(
5019 "declare member outbound taint failed: peer-only or legacy target cannot address registered incarnation {current:?}"
5020 ),
5021 ))
5022 } else {
5023 Ok(())
5024 }
5025 }
5026 };
5027 if let Err((cause, reason)) = incarnation_check {
5028 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5029 return true;
5030 }
5031 let expected_member = match payload.target.as_ref() {
5032 Some(BridgeOutboundTaintTarget::Placed(expected)) => Some(expected),
5033 Some(BridgeOutboundTaintTarget::PeerOnly) | None => None,
5034 };
5035 let effect_authority = match acquire_optional_member_effect_authority(
5036 adapter,
5037 session_id,
5038 expected_member,
5039 "declare member outbound taint",
5040 )
5041 .await
5042 {
5043 Ok(authority) => authority,
5044 Err((cause, reason)) => {
5045 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5046 return true;
5047 }
5048 };
5049 let result = comms_runtime.set_outbound_content_taint(payload.taint);
5054 drop(effect_authority);
5055 match result {
5056 Ok(()) => {
5057 send_bridge_response(
5058 comms_runtime,
5059 candidate,
5060 meerkat_core::interaction::ResponseStatus::Completed,
5061 BridgeReply::Ack(BridgeAck { ok: true }),
5062 None,
5063 )
5064 .await;
5065 }
5066 Err(error) => {
5067 send_bridge_failure(
5068 comms_runtime,
5069 candidate,
5070 BridgeRejectionCause::Internal,
5071 format!("declare member outbound taint failed: {error}"),
5072 None,
5073 )
5074 .await;
5075 }
5076 }
5077 true
5078 }
5079 BridgeCommand::HardCancelMember(payload) => {
5080 serve_hard_cancel_member(
5081 adapter,
5082 session_id,
5083 comms_runtime,
5084 sender,
5085 candidate,
5086 &payload,
5087 )
5088 .await;
5089 true
5090 }
5091 BridgeCommand::CancelTrackedMemberInput(payload) => {
5092 serve_cancel_tracked_member_input(
5093 adapter,
5094 session_id,
5095 comms_runtime,
5096 sender,
5097 candidate,
5098 &payload,
5099 )
5100 .await;
5101 true
5102 }
5103 BridgeCommand::ReadMemberHistory(payload) => {
5104 let sup_payload = BridgeSupervisorPayload {
5107 supervisor: payload.supervisor.clone(),
5108 epoch: payload.epoch,
5109 protocol_version: payload.protocol_version,
5110 };
5111 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
5112 adapter,
5113 session_id,
5114 comms_runtime,
5115 sender,
5116 &sup_payload,
5117 "read member history failed",
5118 )
5119 .await
5120 {
5121 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5122 return true;
5123 }
5124 let observation = match resolve_member_observation_host(adapter) {
5125 Ok(observation) => observation,
5126 Err((cause, reason)) => {
5127 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5128 return true;
5129 }
5130 };
5131 if let Err((cause, reason)) = require_registered_member_incarnation(
5132 adapter,
5133 session_id,
5134 &payload.expected_member,
5135 "read member history",
5136 ) {
5137 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5138 return true;
5139 }
5140 match observation
5141 .read_history(session_id, payload.from_index, payload.limit)
5142 .await
5143 {
5144 Ok(window) => {
5145 if let Err((cause, reason)) = require_registered_member_incarnation(
5146 adapter,
5147 session_id,
5148 &payload.expected_member,
5149 "read member history completion",
5150 ) {
5151 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5152 return true;
5153 }
5154 match bounded_member_history_reply(window) {
5155 Ok(reply) => {
5156 send_bridge_response(
5157 comms_runtime,
5158 candidate,
5159 meerkat_core::interaction::ResponseStatus::Completed,
5160 reply,
5161 None,
5162 )
5163 .await;
5164 }
5165 Err(MemberHistoryReplyError::RowTooLarge {
5166 index,
5167 encoded_bytes,
5168 }) => {
5169 send_bridge_failure(
5170 comms_runtime,
5171 candidate,
5172 BridgeRejectionCause::HistoryRowTooLarge {
5173 index,
5174 encoded_bytes: u64::try_from(encoded_bytes).unwrap_or(u64::MAX),
5175 max_bytes: MEMBER_HISTORY_REPLY_MAX_BYTES as u64,
5176 },
5177 format!(
5178 "member history row at index {index} is {encoded_bytes} bytes (maximum bridge reply budget {MEMBER_HISTORY_REPLY_MAX_BYTES})"
5179 ),
5180 None,
5181 )
5182 .await;
5183 }
5184 Err(MemberHistoryReplyError::Internal(reason)) => {
5185 send_bridge_failure(
5186 comms_runtime,
5187 candidate,
5188 BridgeRejectionCause::Internal,
5189 reason,
5190 None,
5191 )
5192 .await;
5193 }
5194 }
5195 }
5196 Err(error) => {
5197 let (cause, reason) = observation_error_to_bridge_rejection(&error);
5198 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5199 }
5200 }
5201 true
5202 }
5203 BridgeCommand::PollMemberEvents(payload) => {
5204 let sup_payload = BridgeSupervisorPayload {
5212 supervisor: payload.supervisor.clone(),
5213 epoch: payload.epoch,
5214 protocol_version: payload.protocol_version,
5215 };
5216 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
5217 adapter,
5218 session_id,
5219 comms_runtime,
5220 sender,
5221 &sup_payload,
5222 "poll member events failed",
5223 )
5224 .await
5225 {
5226 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5227 return true;
5228 }
5229 if payload.outcome_acks.len() > BRIDGE_TURN_OUTCOME_ACK_MAX {
5230 send_bridge_failure(
5231 comms_runtime,
5232 candidate,
5233 BridgeRejectionCause::Internal,
5234 format!(
5235 "turn-outcome acknowledgement batch has {} rows (maximum {})",
5236 payload.outcome_acks.len(),
5237 BRIDGE_TURN_OUTCOME_ACK_MAX
5238 ),
5239 None,
5240 )
5241 .await;
5242 return true;
5243 }
5244 let observation = match resolve_member_observation_host(adapter) {
5245 Ok(observation) => observation,
5246 Err((cause, reason)) => {
5247 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5248 return true;
5249 }
5250 };
5251 let cursor = match payload.cursor {
5252 BridgeEventCursor::Tail => MemberObservationCursor::Tail,
5253 BridgeEventCursor::At { generation, seq } => {
5254 MemberObservationCursor::At { generation, seq }
5255 }
5256 };
5257 let max = payload
5258 .max
5259 .unwrap_or(MEMBER_POLL_DEFAULT_MAX_ROWS)
5260 .clamp(1, MEMBER_POLL_MAX_ROWS);
5261 let outcome_acks = payload.outcome_acks;
5262 let max_outcomes = payload
5263 .max_outcomes
5264 .unwrap_or(MEMBER_POLL_DEFAULT_MAX_OUTCOMES)
5265 .clamp(1, MEMBER_POLL_MAX_OUTCOMES);
5266 let wait_ms = payload.wait_ms.unwrap_or(0).min(MEMBER_POLL_WAIT_MAX_MS);
5267 let wait = Duration::from_millis(u64::from(wait_ms));
5268 let request = MemberEventsPageRequest {
5269 cursor,
5270 max,
5271 wait,
5272 outcome_acks,
5273 max_outcomes,
5274 expected_member: payload.expected_member,
5275 };
5276 if wait_ms == 0 {
5277 serve_member_events_page(
5279 adapter,
5280 &observation,
5281 session_id,
5282 comms_runtime,
5283 candidate,
5284 request,
5285 )
5286 .await;
5287 return true;
5288 }
5289 let observation = Arc::clone(&observation);
5290 let adapter = Arc::clone(adapter);
5291 let session_id = session_id.clone();
5292 let comms_runtime = Arc::clone(comms_runtime);
5293 let candidate = candidate.clone();
5294 crate::tokio::spawn(async move {
5295 serve_member_events_page(
5296 adapter.as_ref(),
5297 &observation,
5298 &session_id,
5299 &comms_runtime,
5300 &candidate,
5301 request,
5302 )
5303 .await;
5304 });
5305 true
5306 }
5307 BridgeCommand::OpenMemberLiveChannel(payload) => {
5308 let sup_payload = BridgeSupervisorPayload {
5318 supervisor: payload.supervisor.clone(),
5319 epoch: payload.epoch,
5320 protocol_version: payload.protocol_version,
5321 };
5322 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
5323 adapter,
5324 session_id,
5325 comms_runtime,
5326 sender,
5327 &sup_payload,
5328 "open member live channel failed",
5329 )
5330 .await
5331 {
5332 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5333 return true;
5334 }
5335 let live = match resolve_member_live_host(adapter) {
5336 Ok(live) => live,
5337 Err((cause, reason)) => {
5338 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5339 return true;
5340 }
5341 };
5342 let session_id = session_id.clone();
5343 let adapter = Arc::clone(adapter);
5344 let comms_runtime = Arc::clone(comms_runtime);
5345 let candidate = candidate.clone();
5346 let expected_member = payload.expected_member;
5347 let turning_mode = payload.turning_mode;
5348 let transport = payload.transport;
5349 crate::tokio::spawn(async move {
5350 let effect_authority = match acquire_registered_member_live_authority(
5351 adapter.as_ref(),
5352 &session_id,
5353 &expected_member,
5354 "open member live channel",
5355 )
5356 .await
5357 {
5358 Ok(authority) => authority,
5359 Err((cause, reason)) => {
5360 send_bridge_failure(&comms_runtime, &candidate, cause, reason, None).await;
5361 return;
5362 }
5363 };
5364 let result = live.open(&session_id, turning_mode, transport).await;
5365 drop(effect_authority);
5366 match result {
5367 Ok(open) => {
5368 send_bridge_response(
5369 &comms_runtime,
5370 &candidate,
5371 meerkat_core::interaction::ResponseStatus::Completed,
5372 BridgeReply::MemberLiveChannelOpened(BridgeLiveOpenedResponse { open }),
5373 None,
5374 )
5375 .await;
5376 }
5377 Err(error) => {
5378 let (cause, reason) = member_live_error_to_bridge_rejection(&error);
5379 send_bridge_failure(&comms_runtime, &candidate, cause, reason, None).await;
5380 }
5381 }
5382 });
5383 true
5384 }
5385 BridgeCommand::CloseMemberLiveChannel(payload) => {
5386 let sup_payload = BridgeSupervisorPayload {
5391 supervisor: payload.supervisor.clone(),
5392 epoch: payload.epoch,
5393 protocol_version: payload.protocol_version,
5394 };
5395 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
5396 adapter,
5397 session_id,
5398 comms_runtime,
5399 sender,
5400 &sup_payload,
5401 "close member live channel failed",
5402 )
5403 .await
5404 {
5405 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5406 return true;
5407 }
5408 let live = match resolve_member_live_host(adapter) {
5409 Ok(live) => live,
5410 Err((cause, reason)) => {
5411 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5412 return true;
5413 }
5414 };
5415 let effect_authority = match acquire_registered_member_live_authority(
5416 adapter,
5417 session_id,
5418 &payload.expected_member,
5419 "close member live channel",
5420 )
5421 .await
5422 {
5423 Ok(authority) => authority,
5424 Err((cause, reason)) => {
5425 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5426 return true;
5427 }
5428 };
5429 let result = live.close(session_id, &payload.channel_id).await;
5430 drop(effect_authority);
5431 match result {
5432 Ok(status) => {
5433 send_bridge_response(
5434 comms_runtime,
5435 candidate,
5436 meerkat_core::interaction::ResponseStatus::Completed,
5437 BridgeReply::MemberLiveChannelClosed { status },
5438 None,
5439 )
5440 .await;
5441 }
5442 Err(error) => {
5443 let (cause, reason) = member_live_error_to_bridge_rejection(&error);
5444 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5445 }
5446 }
5447 true
5448 }
5449 BridgeCommand::MemberLiveChannelStatus(payload) => {
5450 let sup_payload = BridgeSupervisorPayload {
5455 supervisor: payload.supervisor.clone(),
5456 epoch: payload.epoch,
5457 protocol_version: payload.protocol_version,
5458 };
5459 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
5460 adapter,
5461 session_id,
5462 comms_runtime,
5463 sender,
5464 &sup_payload,
5465 "member live channel status failed",
5466 )
5467 .await
5468 {
5469 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5470 return true;
5471 }
5472 let live = match resolve_member_live_host(adapter) {
5473 Ok(live) => live,
5474 Err((cause, reason)) => {
5475 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5476 return true;
5477 }
5478 };
5479 let effect_authority = match acquire_registered_member_live_authority(
5480 adapter,
5481 session_id,
5482 &payload.expected_member,
5483 "member live channel status",
5484 )
5485 .await
5486 {
5487 Ok(authority) => authority,
5488 Err((cause, reason)) => {
5489 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5490 return true;
5491 }
5492 };
5493 let result = live.status(session_id, payload.channel_id).await;
5494 drop(effect_authority);
5495 match result {
5496 Ok(report) => {
5497 send_bridge_response(
5498 comms_runtime,
5499 candidate,
5500 meerkat_core::interaction::ResponseStatus::Completed,
5501 BridgeReply::MemberLiveChannelStatusReport {
5502 channel_id: report.channel_id,
5503 status: report.status,
5504 },
5505 None,
5506 )
5507 .await;
5508 }
5509 Err(error) => {
5510 let (cause, reason) = member_live_error_to_bridge_rejection(&error);
5511 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5512 }
5513 }
5514 true
5515 }
5516 BridgeCommand::ControlMemberLiveChannel(payload) => {
5517 let sup_payload = BridgeSupervisorPayload {
5524 supervisor: payload.supervisor.clone(),
5525 epoch: payload.epoch,
5526 protocol_version: payload.protocol_version,
5527 };
5528 if let Err((cause, reason)) = resolve_authorized_supervisor_with_response_route(
5529 adapter,
5530 session_id,
5531 comms_runtime,
5532 sender,
5533 &sup_payload,
5534 "control member live channel failed",
5535 )
5536 .await
5537 {
5538 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5539 return true;
5540 }
5541 let live = match resolve_member_live_host(adapter) {
5542 Ok(live) => live,
5543 Err((cause, reason)) => {
5544 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5545 return true;
5546 }
5547 };
5548 let effect_authority = match acquire_registered_member_live_authority(
5549 adapter,
5550 session_id,
5551 &payload.expected_member,
5552 "control member live channel",
5553 )
5554 .await
5555 {
5556 Ok(authority) => authority,
5557 Err((cause, reason)) => {
5558 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5559 return true;
5560 }
5561 };
5562 let result = live
5563 .control(session_id, &payload.channel_id, payload.verb)
5564 .await;
5565 drop(effect_authority);
5566 match result {
5567 Ok(outcome) => {
5568 send_bridge_response(
5569 comms_runtime,
5570 candidate,
5571 meerkat_core::interaction::ResponseStatus::Completed,
5572 BridgeReply::MemberLiveChannelControlled(BridgeLiveControlledResponse {
5573 outcome,
5574 }),
5575 None,
5576 )
5577 .await;
5578 }
5579 Err(error) => {
5580 let (cause, reason) = member_live_error_to_bridge_rejection(&error);
5581 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5582 }
5583 }
5584 true
5585 }
5586 _ => {
5587 send_bridge_failure(
5588 comms_runtime,
5589 candidate,
5590 BridgeRejectionCause::Unsupported,
5591 "unsupported supervisor bridge command".to_string(),
5592 None,
5593 )
5594 .await;
5595 true
5596 }
5597 }
5598}
5599
5600#[derive(Debug)]
5601enum MemberHistoryReplyError {
5602 RowTooLarge { index: u64, encoded_bytes: usize },
5603 Internal(String),
5604}
5605
5606fn bounded_member_history_reply(
5612 window: MemberHistoryWindow,
5613) -> Result<BridgeReply, MemberHistoryReplyError> {
5614 let mut page = BridgeMemberHistoryPage {
5615 generation: window.generation,
5616 page: WireMemberHistoryPageBody::try_from_history_page(&window.page).map_err(|error| {
5617 MemberHistoryReplyError::Internal(format!(
5618 "member history page projection failed: {error}"
5619 ))
5620 })?,
5621 };
5622 let encoded_len = |candidate: &BridgeMemberHistoryPage| {
5623 serde_json::to_vec(&BridgeReply::MemberHistoryPage(candidate.clone()))
5624 .map(|bytes| bytes.len())
5625 .map_err(|error| {
5626 MemberHistoryReplyError::Internal(format!(
5627 "member history reply serialization failed: {error}"
5628 ))
5629 })
5630 };
5631 let full_bytes = encoded_len(&page)?;
5632 if full_bytes <= MEMBER_HISTORY_REPLY_MAX_BYTES {
5633 return Ok(BridgeReply::MemberHistoryPage(page));
5634 }
5635
5636 let original_len = page.page.messages.len();
5637 if original_len == 0 {
5638 return Err(MemberHistoryReplyError::Internal(format!(
5639 "empty member history reply exceeds the {MEMBER_HISTORY_REPLY_MAX_BYTES} byte bridge budget"
5640 )));
5641 }
5642 if original_len == 1 {
5643 return Err(MemberHistoryReplyError::RowTooLarge {
5644 index: page.page.from_index,
5645 encoded_bytes: full_bytes,
5646 });
5647 }
5648
5649 let candidate_with_prefix = |prefix_len: usize| {
5650 let mut candidate = page.clone();
5651 candidate.page.messages.truncate(prefix_len);
5652 if prefix_len < original_len {
5653 let served = u64::try_from(prefix_len).map_err(|_| {
5654 MemberHistoryReplyError::Internal(format!(
5655 "member history prefix length {prefix_len} exceeds the u64 cursor domain"
5656 ))
5657 })?;
5658 candidate.page.next_index = Some(
5659 candidate
5660 .page
5661 .from_index
5662 .checked_add(served)
5663 .ok_or_else(|| {
5664 MemberHistoryReplyError::Internal(format!(
5665 "member history continuation cursor exhausts the u64 domain at {} + {served}",
5666 candidate.page.from_index
5667 ))
5668 })?,
5669 );
5670 candidate.page.complete = false;
5671 }
5672 Ok(candidate)
5673 };
5674
5675 let first = candidate_with_prefix(1)?;
5678 let first_bytes = encoded_len(&first)?;
5679 if first_bytes > MEMBER_HISTORY_REPLY_MAX_BYTES {
5680 return Err(MemberHistoryReplyError::RowTooLarge {
5681 index: page.page.from_index,
5682 encoded_bytes: first_bytes,
5683 });
5684 }
5685
5686 let mut low = 1usize;
5689 let mut high = original_len - 1;
5690 while low < high {
5691 let mid = low + (high - low).div_ceil(2);
5692 let candidate = candidate_with_prefix(mid)?;
5693 if encoded_len(&candidate)? <= MEMBER_HISTORY_REPLY_MAX_BYTES {
5694 low = mid;
5695 } else {
5696 high = mid - 1;
5697 }
5698 }
5699 page = candidate_with_prefix(low)?;
5700 Ok(BridgeReply::MemberHistoryPage(page))
5701}
5702
5703#[derive(Debug)]
5706enum MemberEventsReplyError {
5707 OversizedEvent {
5708 generation: u64,
5709 durable_seq: u64,
5710 encoded_bytes: usize,
5711 },
5712 Internal(String),
5713}
5714
5715fn member_event_frontier_after(sequence: u64) -> Result<u64, MemberEventsReplyError> {
5716 sequence.checked_add(1).ok_or_else(|| {
5717 MemberEventsReplyError::Internal(
5718 "member event sequence space is exhausted; reply cursor cannot advance past u64::MAX"
5719 .to_string(),
5720 )
5721 })
5722}
5723
5724fn bounded_member_events_reply(
5725 window: MemberEventsWindow,
5726) -> Result<BridgeReply, MemberEventsReplyError> {
5727 let MemberEventsWindow {
5728 runtime_incarnation,
5729 generation,
5730 fence_token,
5731 rows,
5732 from_seq,
5733 next_seq,
5734 watermark,
5735 turn_outcomes,
5736 outcomes_complete,
5737 } = window;
5738 let mut page = BridgeMemberEventsPage {
5739 runtime_incarnation,
5740 generation,
5741 fence_token,
5742 events: rows
5743 .into_iter()
5744 .map(|(durable_seq, envelope)| WireEventRow {
5745 durable_seq,
5746 envelope,
5747 })
5748 .collect(),
5749 from_seq,
5750 next_seq,
5751 watermark,
5752 turn_outcomes,
5753 outcomes_complete,
5754 };
5755
5756 if let Some(last) = page.events.last() {
5761 let expected_next = member_event_frontier_after(last.durable_seq)?;
5762 if page.next_seq != expected_next {
5763 return Err(MemberEventsReplyError::Internal(format!(
5764 "member event page next_seq {} does not exactly follow last durable seq {}",
5765 page.next_seq, last.durable_seq
5766 )));
5767 }
5768 } else if page.watermark == u64::MAX && page.next_seq == u64::MAX {
5769 return Err(MemberEventsReplyError::Internal(
5770 "member event sequence space is exhausted at an empty MAX frontier".to_string(),
5771 ));
5772 }
5773
5774 let encoded_len = |candidate: &BridgeMemberEventsPage| {
5775 serde_json::to_vec(&BridgeReply::MemberEventsPage(candidate.clone()))
5776 .map(|bytes| bytes.len())
5777 .map_err(|error| {
5778 MemberEventsReplyError::Internal(format!(
5779 "member events reply serialization failed: {error}"
5780 ))
5781 })
5782 };
5783 if encoded_len(&page)? <= MEMBER_EVENTS_REPLY_MAX_BYTES {
5784 return Ok(BridgeReply::MemberEventsPage(page));
5785 }
5786
5787 if !page.events.is_empty() {
5791 let mut first_only = page.clone();
5792 first_only.events.truncate(1);
5793 first_only.next_seq = member_event_frontier_after(first_only.events[0].durable_seq)?;
5794 first_only.turn_outcomes.clear();
5795 first_only.outcomes_complete = true;
5796 let event_only_bytes = encoded_len(&first_only)?;
5797 if event_only_bytes > MEMBER_EVENTS_REPLY_MAX_BYTES {
5798 return Err(MemberEventsReplyError::OversizedEvent {
5799 generation: page.generation,
5800 durable_seq: page.events[0].durable_seq,
5801 encoded_bytes: event_only_bytes,
5802 });
5803 }
5804 }
5805
5806 let mut low = 0usize;
5810 let mut high = page.events.len();
5811 while low < high {
5812 let mid = low + (high - low).div_ceil(2);
5813 let mut candidate = page.clone();
5814 candidate.events.truncate(mid);
5815 candidate.next_seq = match candidate.events.last() {
5816 Some(row) => member_event_frontier_after(row.durable_seq)?,
5817 None => from_seq,
5818 };
5819 candidate.turn_outcomes.clear();
5820 candidate.outcomes_complete = false;
5821 if encoded_len(&candidate)? <= MEMBER_EVENTS_REPLY_MAX_BYTES {
5822 low = mid;
5823 } else {
5824 high = mid - 1;
5825 }
5826 }
5827 page.events.truncate(low);
5828 page.next_seq = match page.events.last() {
5829 Some(row) => member_event_frontier_after(row.durable_seq)?,
5830 None => from_seq,
5831 };
5832
5833 let original_outcome_len = page.turn_outcomes.len();
5837 page.turn_outcomes
5838 .retain(|record| record.terminal_seq < page.next_seq);
5839 let eligible_outcome_len = page.turn_outcomes.len();
5840 page.outcomes_complete = page.outcomes_complete && eligible_outcome_len == original_outcome_len;
5841 low = 0;
5842 high = eligible_outcome_len;
5843 while low < high {
5844 let mid = low + (high - low).div_ceil(2);
5845 let mut candidate = page.clone();
5846 candidate.turn_outcomes.truncate(mid);
5847 candidate.outcomes_complete = page.outcomes_complete && mid == eligible_outcome_len;
5848 if encoded_len(&candidate)? <= MEMBER_EVENTS_REPLY_MAX_BYTES {
5849 low = mid;
5850 } else {
5851 high = mid - 1;
5852 }
5853 }
5854 if eligible_outcome_len > 0 && low == 0 && page.events.is_empty() {
5855 return Err(MemberEventsReplyError::Internal(format!(
5856 "one bounded turn outcome cannot fit the {MEMBER_EVENTS_REPLY_MAX_BYTES} byte bridge budget"
5857 )));
5858 }
5859 page.turn_outcomes.truncate(low);
5860 if low < eligible_outcome_len {
5861 page.outcomes_complete = false;
5862 }
5863 Ok(BridgeReply::MemberEventsPage(page))
5864}
5865
5866struct MemberEventsPageRequest {
5867 cursor: MemberObservationCursor,
5868 max: u32,
5869 wait: Duration,
5870 outcome_acks: Vec<meerkat_contracts::wire::supervisor_bridge::BridgeTurnOutcomeAck>,
5871 max_outcomes: u32,
5872 expected_member: BridgeMemberIncarnation,
5873}
5874
5875async fn serve_member_events_page(
5876 adapter: &MeerkatMachine,
5877 observation: &Arc<dyn MemberObservationHost>,
5878 session_id: &SessionId,
5879 comms_runtime: &Arc<dyn CommsRuntime>,
5880 candidate: &PeerInputCandidate,
5881 request: MemberEventsPageRequest,
5882) {
5883 let MemberEventsPageRequest {
5884 cursor,
5885 max,
5886 wait,
5887 outcome_acks,
5888 max_outcomes,
5889 expected_member,
5890 } = request;
5891 if let Err((cause, reason)) = require_registered_member_incarnation(
5892 adapter,
5893 session_id,
5894 &expected_member,
5895 "poll member events",
5896 ) {
5897 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5898 return;
5899 }
5900 match observation
5901 .poll_events(
5902 session_id,
5903 MemberEventsPollRequest {
5904 expected_member: &expected_member,
5905 cursor,
5906 max,
5907 wait,
5908 outcome_acks: &outcome_acks,
5909 max_outcomes,
5910 },
5911 )
5912 .await
5913 {
5914 Ok(window) => {
5915 if let Err((cause, reason)) = require_registered_member_incarnation(
5916 adapter,
5917 session_id,
5918 &expected_member,
5919 "poll member events completion",
5920 ) {
5921 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5922 return;
5923 }
5924 match bounded_member_events_reply(window) {
5925 Ok(reply) => {
5926 send_bridge_response(
5927 comms_runtime,
5928 candidate,
5929 meerkat_core::interaction::ResponseStatus::Completed,
5930 reply,
5931 None,
5932 )
5933 .await;
5934 }
5935 Err(MemberEventsReplyError::OversizedEvent {
5936 generation,
5937 durable_seq,
5938 encoded_bytes,
5939 }) => {
5940 let Some(next_seq) = durable_seq.checked_add(1) else {
5941 send_bridge_failure(
5942 comms_runtime,
5943 candidate,
5944 BridgeRejectionCause::Internal,
5945 "member event sequence space is exhausted; oversized-row disposition cannot advance the cursor".to_string(),
5946 None,
5947 )
5948 .await;
5949 return;
5950 };
5951 send_bridge_failure(
5952 comms_runtime,
5953 candidate,
5954 BridgeRejectionCause::OversizedEvent {
5955 generation,
5956 durable_seq,
5957 next_seq,
5958 encoded_bytes: u64::try_from(encoded_bytes).unwrap_or(u64::MAX),
5959 max_bytes: MEMBER_EVENTS_REPLY_MAX_BYTES as u64,
5960 },
5961 format!(
5962 "single member event row at durable seq {durable_seq} is {encoded_bytes} bytes (maximum bridge reply budget {MEMBER_EVENTS_REPLY_MAX_BYTES})"
5963 ),
5964 None,
5965 )
5966 .await;
5967 }
5968 Err(MemberEventsReplyError::Internal(reason)) => {
5969 send_bridge_failure(
5970 comms_runtime,
5971 candidate,
5972 BridgeRejectionCause::Internal,
5973 reason,
5974 None,
5975 )
5976 .await;
5977 }
5978 }
5979 }
5980 Err(error) => {
5981 let (cause, reason) = observation_error_to_bridge_rejection(&error);
5982 send_bridge_failure(comms_runtime, candidate, cause, reason, None).await;
5983 }
5984 }
5985}
5986
5987fn directed_rejection_after_pending_cancel(
6011 input_id: &str,
6012 cause: BridgeDeliveryRejectionCause,
6013 reason: String,
6014 cancellation: Result<(), crate::member_observation::DirectedTurnReject>,
6015) -> Result<BridgeDeliveryResponse, String> {
6016 cancellation.map_err(|error| {
6017 format!(
6018 "runtime proved no admission, but durable Pending cancellation failed: {}",
6019 error.detail
6020 )
6021 })?;
6022 Ok(BridgeDeliveryResponse {
6023 input_id: input_id.to_string(),
6024 canonical_input_id: None,
6025 outcome: BridgeDeliveryOutcome::Rejected { cause, reason },
6026 })
6027}
6028
6029fn definite_no_effect_delivery_rejection(
6030 error: &crate::traits::RuntimeDriverError,
6031 current_member: Option<BridgeMemberIncarnation>,
6032 expected_member: &BridgeMemberIncarnation,
6033) -> Option<BridgeDeliveryRejectionCause> {
6034 match error {
6035 crate::traits::RuntimeDriverError::ValidationFailed { reason } => {
6036 Some(BridgeDeliveryRejectionCause::Internal {
6037 detail: reason.clone(),
6038 })
6039 }
6040 crate::traits::RuntimeDriverError::StaleAuthority { .. } => {
6041 Some(BridgeDeliveryRejectionCause::StaleMemberResidency {
6042 expected: expected_member.clone(),
6043 current: current_member,
6044 })
6045 }
6046 _ => None,
6047 }
6048}
6049
6050#[cfg(test)]
6055async fn open_directed_turn_before_runtime_accept<T, F, Fut>(
6056 observation: &Arc<dyn MemberObservationHost>,
6057 session_id: &SessionId,
6058 expected_member: &BridgeMemberIncarnation,
6059 input_id: &str,
6060 accept: F,
6061) -> Result<
6062 (crate::member_observation::DirectedTurnWindow, T),
6063 crate::member_observation::DirectedTurnReject,
6064>
6065where
6066 F: FnOnce() -> Fut,
6067 Fut: std::future::Future<Output = T>,
6068{
6069 let window = observation
6070 .open_directed_turn_window(session_id, expected_member, input_id)
6071 .await?;
6072 Ok((window, accept().await))
6073}
6074
6075enum DirectedTurnPreAcceptDecision {
6076 SubmitToRuntime,
6077 TerminalReplay(BridgeDeliveryResponse),
6078}
6079
6080fn directed_turn_pre_accept_decision(
6086 request_input_id: &str,
6087 expected_member: &BridgeMemberIncarnation,
6088 window: &DirectedTurnWindow,
6089) -> Result<DirectedTurnPreAcceptDecision, String> {
6090 if matches!(
6091 window.tracking,
6092 DirectedTurnTracking::PendingFresh | DirectedTurnTracking::PendingReplay
6093 ) {
6094 return Ok(DirectedTurnPreAcceptDecision::SubmitToRuntime);
6095 }
6096 if window.input_id != request_input_id || &window.expected_member != expected_member {
6097 return Err(format!(
6098 "terminal replay reservation does not match request input/residency: request input '{request_input_id}', reserved input '{}', request member {expected_member:?}, reserved member {:?}",
6099 window.input_id, window.expected_member
6100 ));
6101 }
6102 Ok(DirectedTurnPreAcceptDecision::TerminalReplay(
6103 BridgeDeliveryResponse {
6104 input_id: request_input_id.to_string(),
6105 canonical_input_id: Some(request_input_id.to_string()),
6106 outcome: BridgeDeliveryOutcome::Deduplicated {
6107 existing_input_id: request_input_id.to_string(),
6108 },
6109 },
6110 ))
6111}
6112
6113async fn serve_tracked_member_delivery(
6114 adapter: &Arc<MeerkatMachine>,
6115 session_id: &SessionId,
6116 comms_runtime: &Arc<dyn CommsRuntime>,
6117 candidate: &PeerInputCandidate,
6118 sender_peer_id: PeerId,
6119 payload: BridgeDeliveryPayload,
6120) {
6121 let request_input_id = payload.input_id.clone();
6122 let reject = |detail: String| BridgeDeliveryResponse {
6123 input_id: request_input_id.clone(),
6124 canonical_input_id: None,
6125 outcome: BridgeDeliveryOutcome::Rejected {
6126 cause: BridgeDeliveryRejectionCause::TurnDirectiveUnsupported {
6127 detail: detail.clone(),
6128 },
6129 reason: detail,
6130 },
6131 };
6132 if let Err(detail) = validate_delivery_tracking_request(&payload) {
6133 let response = reject(detail);
6134 send_bridge_response(
6135 comms_runtime,
6136 candidate,
6137 meerkat_core::interaction::ResponseStatus::Completed,
6138 BridgeReply::Delivery(response),
6139 None,
6140 )
6141 .await;
6142 return;
6143 }
6144 let Some(expected_member) = payload.expected_member.clone() else {
6145 let response =
6146 reject("tracked delivery requires an exact placed-member incarnation".to_string());
6147 send_bridge_response(
6148 comms_runtime,
6149 candidate,
6150 meerkat_core::interaction::ResponseStatus::Completed,
6151 BridgeReply::Delivery(response),
6152 None,
6153 )
6154 .await;
6155 return;
6156 };
6157 let input = if let Some(directive) = payload.turn.clone() {
6158 if !payload.injected_context.is_empty() {
6159 let response = reject(
6163 "injected_context is not supported on directive-bearing deliveries".to_string(),
6164 );
6165 send_bridge_response(
6166 comms_runtime,
6167 candidate,
6168 meerkat_core::interaction::ResponseStatus::Completed,
6169 BridgeReply::Delivery(response),
6170 None,
6171 )
6172 .await;
6173 return;
6174 }
6175 if payload.objective_id.is_some() {
6176 let response =
6180 reject("objective_id is not supported on directive-bearing deliveries".to_string());
6181 send_bridge_response(
6182 comms_runtime,
6183 candidate,
6184 meerkat_core::interaction::ResponseStatus::Completed,
6185 BridgeReply::Delivery(response),
6186 None,
6187 )
6188 .await;
6189 return;
6190 }
6191 let mut turn_metadata = crate::runtime_loop::for_bridge_turn_directive(
6192 payload.handling_mode,
6193 directive.tool_overlay.clone().map(Into::into),
6194 );
6195 turn_metadata.transient_turn_context = payload.transient_turn_context.clone();
6196 match crate::mob_adapter::create_tracked_flow_step_input(
6197 &directive.correlation.step_id,
6198 payload.content.clone(),
6199 &directive.correlation.run_id,
6200 Some(turn_metadata),
6201 &payload.input_id,
6202 ) {
6203 Ok(input) => input,
6204 Err(detail) => {
6205 let response = reject(detail);
6206 send_bridge_response(
6207 comms_runtime,
6208 candidate,
6209 meerkat_core::interaction::ResponseStatus::Completed,
6210 BridgeReply::Delivery(response),
6211 None,
6212 )
6213 .await;
6214 return;
6215 }
6216 }
6217 } else {
6218 match peer_input_from_delivery_payload(session_id, sender_peer_id, payload) {
6219 Ok(input) => input,
6220 Err(reason) => {
6221 let response = BridgeDeliveryResponse {
6222 input_id: request_input_id.clone(),
6223 canonical_input_id: None,
6224 outcome: BridgeDeliveryOutcome::Rejected {
6225 cause: BridgeDeliveryRejectionCause::Internal {
6226 detail: reason.clone(),
6227 },
6228 reason,
6229 },
6230 };
6231 send_bridge_response(
6232 comms_runtime,
6233 candidate,
6234 meerkat_core::interaction::ResponseStatus::Completed,
6235 BridgeReply::Delivery(response),
6236 None,
6237 )
6238 .await;
6239 return;
6240 }
6241 }
6242 };
6243 let Some(observation) = adapter.member_observation_host() else {
6244 let response = reject("member host serves no observation substrate".to_string());
6245 send_bridge_response(
6246 comms_runtime,
6247 candidate,
6248 meerkat_core::interaction::ResponseStatus::Completed,
6249 BridgeReply::Delivery(response),
6250 None,
6251 )
6252 .await;
6253 return;
6254 };
6255 let window = match observation
6260 .open_directed_turn_window(session_id, &expected_member, &request_input_id)
6261 .await
6262 {
6263 Ok(window) => window,
6264 Err(rejected) => {
6265 if !rejected.definite_no_effect {
6266 send_bridge_failure(
6267 comms_runtime,
6268 candidate,
6269 BridgeRejectionCause::Internal,
6270 rejected.detail,
6271 None,
6272 )
6273 .await;
6274 return;
6275 }
6276 let response = BridgeDeliveryResponse {
6277 input_id: request_input_id.clone(),
6278 canonical_input_id: None,
6279 outcome: BridgeDeliveryOutcome::Rejected {
6280 cause: rejected.cause,
6281 reason: rejected.detail,
6282 },
6283 };
6284 send_bridge_response(
6285 comms_runtime,
6286 candidate,
6287 meerkat_core::interaction::ResponseStatus::Completed,
6288 BridgeReply::Delivery(response),
6289 None,
6290 )
6291 .await;
6292 return;
6293 }
6294 };
6295 if window.expected_member != expected_member {
6296 let cancellation = observation
6297 .cancel_directed_turn_window(session_id, window)
6298 .await;
6299 let reason = format!(
6300 "directed-turn residency changed before accept; controller expected {expected_member:?}"
6301 );
6302 match directed_rejection_after_pending_cancel(
6303 &request_input_id,
6304 BridgeDeliveryRejectionCause::StaleMemberResidency {
6305 expected: expected_member.clone(),
6306 current: adapter.member_incarnation(session_id),
6307 },
6308 reason,
6309 cancellation,
6310 ) {
6311 Ok(response) => {
6312 send_bridge_response(
6313 comms_runtime,
6314 candidate,
6315 meerkat_core::interaction::ResponseStatus::Completed,
6316 BridgeReply::Delivery(response),
6317 None,
6318 )
6319 .await;
6320 }
6321 Err(detail) => {
6322 send_bridge_failure(
6323 comms_runtime,
6324 candidate,
6325 BridgeRejectionCause::Internal,
6326 detail,
6327 None,
6328 )
6329 .await;
6330 }
6331 }
6332 return;
6333 }
6334 match directed_turn_pre_accept_decision(&request_input_id, &expected_member, &window) {
6335 Ok(DirectedTurnPreAcceptDecision::SubmitToRuntime) => {}
6336 Ok(DirectedTurnPreAcceptDecision::TerminalReplay(response)) => {
6337 send_bridge_response(
6338 comms_runtime,
6339 candidate,
6340 meerkat_core::interaction::ResponseStatus::Completed,
6341 BridgeReply::Delivery(response),
6342 None,
6343 )
6344 .await;
6345 return;
6346 }
6347 Err(detail) => {
6348 send_bridge_failure(
6349 comms_runtime,
6350 candidate,
6351 BridgeRejectionCause::Internal,
6352 detail,
6353 None,
6354 )
6355 .await;
6356 return;
6357 }
6358 }
6359 let Some(journal) = adapter.tracked_turn_journal(session_id) else {
6360 let cancellation = observation
6361 .cancel_directed_turn_window(session_id, window)
6362 .await;
6363 let response = match directed_rejection_after_pending_cancel(
6364 &request_input_id,
6365 BridgeDeliveryRejectionCause::TurnDirectiveUnsupported {
6366 detail: "member runtime has no tracked-turn journal registration".to_string(),
6367 },
6368 "member runtime has no tracked-turn journal registration".to_string(),
6369 cancellation,
6370 ) {
6371 Ok(response) => response,
6372 Err(detail) => {
6373 send_bridge_failure(
6374 comms_runtime,
6375 candidate,
6376 BridgeRejectionCause::Internal,
6377 detail,
6378 None,
6379 )
6380 .await;
6381 return;
6382 }
6383 };
6384 send_bridge_response(
6385 comms_runtime,
6386 candidate,
6387 meerkat_core::interaction::ResponseStatus::Completed,
6388 BridgeReply::Delivery(response),
6389 None,
6390 )
6391 .await;
6392 return;
6393 };
6394 if journal.member_incarnation() != &expected_member {
6395 let cancellation = observation
6396 .cancel_directed_turn_window(session_id, window)
6397 .await;
6398 let detail = "tracked-turn journal incarnation differs from reserved residency".to_string();
6399 match directed_rejection_after_pending_cancel(
6400 &request_input_id,
6401 BridgeDeliveryRejectionCause::TurnDirectiveUnsupported {
6402 detail: detail.clone(),
6403 },
6404 detail,
6405 cancellation,
6406 ) {
6407 Ok(response) => {
6408 send_bridge_response(
6409 comms_runtime,
6410 candidate,
6411 meerkat_core::interaction::ResponseStatus::Completed,
6412 BridgeReply::Delivery(response),
6413 None,
6414 )
6415 .await;
6416 }
6417 Err(detail) => {
6418 send_bridge_failure(
6419 comms_runtime,
6420 candidate,
6421 BridgeRejectionCause::Internal,
6422 detail,
6423 None,
6424 )
6425 .await;
6426 }
6427 }
6428 return;
6429 }
6430 let _admission_permit = match observation
6436 .lock_and_revalidate_directed_turn_admission(session_id, window.admission_request())
6437 .await
6438 {
6439 Ok(crate::member_observation::DirectedTurnAdmissionDecision::Admit(permit)) => permit,
6440 Ok(crate::member_observation::DirectedTurnAdmissionDecision::TerminalReplay) => {
6441 let response = BridgeDeliveryResponse {
6442 input_id: request_input_id.clone(),
6443 canonical_input_id: Some(request_input_id.clone()),
6444 outcome: BridgeDeliveryOutcome::Deduplicated {
6445 existing_input_id: request_input_id.clone(),
6446 },
6447 };
6448 send_bridge_response(
6449 comms_runtime,
6450 candidate,
6451 meerkat_core::interaction::ResponseStatus::Completed,
6452 BridgeReply::Delivery(response),
6453 None,
6454 )
6455 .await;
6456 return;
6457 }
6458 Err(error) => {
6459 send_bridge_failure(
6460 comms_runtime,
6461 candidate,
6462 BridgeRejectionCause::Internal,
6463 format!(
6464 "directed-turn final admission revalidation was ambiguous: {}",
6465 error.detail
6466 ),
6467 None,
6468 )
6469 .await;
6470 return;
6471 }
6472 };
6473 let accept_result = adapter
6474 .accept_input_with_completion_for_member_residency(
6475 session_id,
6476 input,
6477 Some(&expected_member),
6478 )
6479 .await;
6480 match accept_result {
6481 Ok((accept_outcome, completion_handle)) => {
6482 let response = match accept_outcome {
6483 crate::accept::AcceptOutcome::Accepted { input_id, .. } => {
6484 let canonical_input_id = input_id.to_string();
6485 if let Err(rejected) = observation
6486 .admit_directed_turn(
6487 session_id,
6488 DirectedTurnAdmission {
6489 input_id: request_input_id.clone(),
6490 window,
6491 completion_handle,
6492 journal,
6493 },
6494 )
6495 .await
6496 {
6497 tracing::error!(
6501 session_id = %session_id,
6502 input_id = %request_input_id,
6503 detail = %rejected.detail,
6504 "directed turn accepted but watcher admission failed; durable Pending retained for recovery"
6505 );
6506 }
6507 BridgeDeliveryResponse {
6508 input_id: request_input_id.clone(),
6509 canonical_input_id: Some(canonical_input_id),
6510 outcome: BridgeDeliveryOutcome::Accepted,
6511 }
6512 }
6513 crate::accept::AcceptOutcome::Deduplicated { existing_id, .. } => {
6514 let existing_id = existing_id.to_string();
6515 if let Err(rejected) = observation
6516 .admit_directed_turn(
6517 session_id,
6518 DirectedTurnAdmission {
6519 input_id: request_input_id.clone(),
6520 window,
6521 completion_handle,
6522 journal,
6523 },
6524 )
6525 .await
6526 {
6527 tracing::error!(
6528 session_id = %session_id,
6529 input_id = %request_input_id,
6530 detail = %rejected.detail,
6531 "deduplicated directed turn could not reattach watcher; durable Pending retained for recovery"
6532 );
6533 }
6534 BridgeDeliveryResponse {
6535 input_id: request_input_id.clone(),
6536 canonical_input_id: Some(existing_id.clone()),
6537 outcome: BridgeDeliveryOutcome::Deduplicated {
6538 existing_input_id: existing_id,
6539 },
6540 }
6541 }
6542 crate::accept::AcceptOutcome::Rejected { reason } => {
6543 let cause = bridge_delivery_rejection_cause(&reason);
6544 let cancellation = observation
6545 .cancel_directed_turn_window(session_id, window)
6546 .await;
6547 match directed_rejection_after_pending_cancel(
6548 &request_input_id,
6549 cause,
6550 reason.to_string(),
6551 cancellation,
6552 ) {
6553 Ok(response) => response,
6554 Err(detail) => {
6555 tracing::error!(
6556 session_id = %session_id,
6557 input_id = %request_input_id,
6558 detail = %detail,
6559 "proven-rejected directed turn could not durably certify no-effect"
6560 );
6561 send_bridge_failure(
6562 comms_runtime,
6563 candidate,
6564 BridgeRejectionCause::Internal,
6565 detail,
6566 None,
6567 )
6568 .await;
6569 return;
6570 }
6571 }
6572 }
6573 };
6574 send_bridge_response(
6575 comms_runtime,
6576 candidate,
6577 meerkat_core::interaction::ResponseStatus::Completed,
6578 BridgeReply::Delivery(response),
6579 None,
6580 )
6581 .await;
6582 }
6583 Err(error) => {
6584 if let Some(cause) = definite_no_effect_delivery_rejection(
6588 &error,
6589 adapter.member_incarnation(session_id),
6590 &expected_member,
6591 ) {
6592 let cancellation = observation
6593 .cancel_directed_turn_window(session_id, window)
6594 .await;
6595 match directed_rejection_after_pending_cancel(
6596 &request_input_id,
6597 cause,
6598 error.to_string(),
6599 cancellation,
6600 ) {
6601 Ok(response) => {
6602 send_bridge_response(
6603 comms_runtime,
6604 candidate,
6605 meerkat_core::interaction::ResponseStatus::Completed,
6606 BridgeReply::Delivery(response),
6607 None,
6608 )
6609 .await;
6610 }
6611 Err(detail) => {
6612 tracing::error!(
6613 session_id = %session_id,
6614 input_id = %request_input_id,
6615 detail = %detail,
6616 "proven validation rejection could not durably certify no-effect"
6617 );
6618 send_bridge_failure(
6619 comms_runtime,
6620 candidate,
6621 BridgeRejectionCause::Internal,
6622 detail,
6623 None,
6624 )
6625 .await;
6626 }
6627 }
6628 return;
6629 }
6630 send_bridge_failure(
6631 comms_runtime,
6632 candidate,
6633 BridgeRejectionCause::Internal,
6634 format!("deliver member input failed: {error}"),
6635 None,
6636 )
6637 .await;
6638 }
6639 }
6640}
6641
6642fn runtime_state_to_bridge(state: crate::RuntimeState) -> BridgeMemberRuntimeState {
6644 match state {
6645 crate::RuntimeState::Initializing => BridgeMemberRuntimeState::Initializing,
6646 crate::RuntimeState::Idle => BridgeMemberRuntimeState::Idle,
6647 crate::RuntimeState::Attached => BridgeMemberRuntimeState::Attached,
6648 crate::RuntimeState::Running => BridgeMemberRuntimeState::Running,
6649 crate::RuntimeState::Retired => BridgeMemberRuntimeState::Retired,
6650 crate::RuntimeState::Stopped => BridgeMemberRuntimeState::Stopped,
6651 crate::RuntimeState::Destroyed => BridgeMemberRuntimeState::Destroyed,
6652 }
6653}
6654
6655#[cfg(test)]
6656#[allow(clippy::items_after_test_module)]
6657#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
6658mod tests {
6659 use super::*;
6660 use meerkat_contracts::wire::supervisor_bridge::{
6661 supervisor_bridge_current_protocol_version, supervisor_bridge_default_protocol_version,
6662 supervisor_bridge_supported_protocol_versions,
6663 };
6664 use meerkat_contracts::{
6665 BridgeMobPeerOverlayHandoff, BridgeOutboundTaintPayload, BridgePeerWiringPayload,
6666 };
6667 use meerkat_core::InteractionId;
6668 use meerkat_core::SendError;
6669 use meerkat_core::interaction::InboxInteraction;
6670 use meerkat_core::interaction::{PeerIngressConvention, PeerIngressIdentity};
6671 use meerkat_core::types::HandlingMode;
6672 use serde_json::json;
6673 use std::collections::{HashMap, HashSet};
6674 use uuid::Uuid;
6675
6676 struct ExhaustedDirectedTurnObservation;
6677
6678 #[async_trait::async_trait]
6679 impl MemberObservationHost for ExhaustedDirectedTurnObservation {
6680 async fn member_generation(
6681 &self,
6682 _session: &SessionId,
6683 ) -> Result<u64, MemberObservationError> {
6684 Err(MemberObservationError::Internal {
6685 reason: "unused test seam".to_string(),
6686 })
6687 }
6688
6689 async fn read_history(
6690 &self,
6691 _session: &SessionId,
6692 _from_index: Option<u64>,
6693 _limit: Option<u32>,
6694 ) -> Result<MemberHistoryWindow, MemberObservationError> {
6695 Err(MemberObservationError::Internal {
6696 reason: "unused test seam".to_string(),
6697 })
6698 }
6699
6700 async fn poll_events(
6701 &self,
6702 _session: &SessionId,
6703 _request: MemberEventsPollRequest<'_>,
6704 ) -> Result<MemberEventsWindow, MemberObservationError> {
6705 Err(MemberObservationError::Internal {
6706 reason: "unused test seam".to_string(),
6707 })
6708 }
6709
6710 async fn open_directed_turn_window(
6711 &self,
6712 _session: &SessionId,
6713 _expected_member: &BridgeMemberIncarnation,
6714 _input_id: &str,
6715 ) -> Result<
6716 crate::member_observation::DirectedTurnWindow,
6717 crate::member_observation::DirectedTurnReject,
6718 > {
6719 Err(crate::member_observation::DirectedTurnReject::unsupported(
6720 "member event sequence space is exhausted before Pending reservation",
6721 ))
6722 }
6723
6724 async fn cancel_directed_turn_window(
6725 &self,
6726 _session: &SessionId,
6727 _window: crate::member_observation::DirectedTurnWindow,
6728 ) -> Result<(), crate::member_observation::DirectedTurnReject> {
6729 Err(crate::member_observation::DirectedTurnReject::unsupported(
6730 "unused test seam",
6731 ))
6732 }
6733
6734 async fn admit_directed_turn(
6735 &self,
6736 _session: &SessionId,
6737 _admission: DirectedTurnAdmission,
6738 ) -> Result<(), crate::member_observation::DirectedTurnReject> {
6739 Err(crate::member_observation::DirectedTurnReject::unsupported(
6740 "unused test seam",
6741 ))
6742 }
6743 }
6744
6745 #[tokio::test]
6746 async fn rejected_exhausted_window_never_invokes_runtime_accept() {
6747 let observation: Arc<dyn MemberObservationHost> =
6748 Arc::new(ExhaustedDirectedTurnObservation);
6749 let session_id = SessionId::new();
6750 let accept_calls = std::sync::atomic::AtomicUsize::new(0);
6751 let result = open_directed_turn_before_runtime_accept(
6752 &observation,
6753 &session_id,
6754 &BridgeMemberIncarnation::default(),
6755 "input-exhausted",
6756 || async {
6757 accept_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6758 },
6759 )
6760 .await;
6761 assert!(result.is_err());
6762 assert_eq!(
6763 accept_calls.load(std::sync::atomic::Ordering::SeqCst),
6764 0,
6765 "runtime acceptance must remain strictly downstream of window reservation"
6766 );
6767 }
6768
6769 #[test]
6770 fn exact_terminal_replay_never_invokes_runtime_accept() {
6771 let expected_member = BridgeMemberIncarnation {
6772 mob_id: "mob".to_string(),
6773 agent_identity: "worker".to_string(),
6774 host_id: "host".to_string(),
6775 binding_generation: 3,
6776 member_session_id: SessionId::new().to_string(),
6777 generation: 7,
6778 fence_token: 11,
6779 };
6780 let input_id = Uuid::new_v4().to_string();
6781 let window = DirectedTurnWindow {
6782 expected_member: expected_member.clone(),
6783 subscription: Box::pin(futures::stream::empty()),
6784 window_start: 42,
6785 generation: expected_member.generation,
6786 fence_token: expected_member.fence_token,
6787 input_id: input_id.clone(),
6788 tracking: DirectedTurnTracking::TerminalReplay,
6789 };
6790 let accept_calls = std::sync::atomic::AtomicUsize::new(0);
6791 let response = match directed_turn_pre_accept_decision(&input_id, &expected_member, &window)
6792 .expect("exact terminal replay is valid")
6793 {
6794 DirectedTurnPreAcceptDecision::SubmitToRuntime => {
6795 accept_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6796 panic!("durable terminal replay must dominate runtime admission")
6797 }
6798 DirectedTurnPreAcceptDecision::TerminalReplay(response) => response,
6799 };
6800
6801 assert_eq!(
6802 accept_calls.load(std::sync::atomic::Ordering::SeqCst),
6803 0,
6804 "terminal replay must never reach the runtime accept closure"
6805 );
6806 assert_eq!(response.input_id, input_id);
6807 assert_eq!(
6808 response.canonical_input_id.as_deref(),
6809 Some(input_id.as_str())
6810 );
6811 assert!(matches!(
6812 response.outcome,
6813 BridgeDeliveryOutcome::Deduplicated {
6814 existing_input_id
6815 } if existing_input_id == input_id
6816 ));
6817 }
6818
6819 #[test]
6820 fn terminal_replay_with_different_request_identity_fails_closed() {
6821 let expected_member = BridgeMemberIncarnation {
6822 mob_id: "mob".to_string(),
6823 agent_identity: "worker".to_string(),
6824 host_id: "host".to_string(),
6825 binding_generation: 3,
6826 member_session_id: SessionId::new().to_string(),
6827 generation: 7,
6828 fence_token: 11,
6829 };
6830 let window = DirectedTurnWindow {
6831 expected_member: expected_member.clone(),
6832 subscription: Box::pin(futures::stream::empty()),
6833 window_start: 42,
6834 generation: expected_member.generation,
6835 fence_token: expected_member.fence_token,
6836 input_id: Uuid::new_v4().to_string(),
6837 tracking: DirectedTurnTracking::TerminalReplay,
6838 };
6839 assert!(
6840 directed_turn_pre_accept_decision(
6841 &Uuid::new_v4().to_string(),
6842 &expected_member,
6843 &window,
6844 )
6845 .is_err(),
6846 "a terminal row for another input must never suppress or submit this request"
6847 );
6848 }
6849
6850 #[test]
6851 fn directed_no_effect_rejection_requires_durable_pending_cancel() {
6852 let cause = BridgeDeliveryRejectionCause::Internal {
6853 detail: "validation failed".to_string(),
6854 };
6855 let response = directed_rejection_after_pending_cancel(
6856 "input-1",
6857 cause.clone(),
6858 "Input validation failed: invalid".to_string(),
6859 Ok(()),
6860 )
6861 .expect("durably canceled Pending certifies definite no-effect");
6862 assert!(matches!(
6863 response.outcome,
6864 BridgeDeliveryOutcome::Rejected {
6865 cause: actual,
6866 ..
6867 } if actual == cause
6868 ));
6869
6870 let ambiguous = directed_rejection_after_pending_cancel(
6871 "input-1",
6872 cause,
6873 "Input validation failed: invalid".to_string(),
6874 Err(crate::member_observation::DirectedTurnReject::unsupported(
6875 "host actor cancellation unavailable",
6876 )),
6877 )
6878 .expect_err("cancel failure must not mint a definite Delivery Rejected");
6879 assert!(ambiguous.contains("durable Pending cancellation failed"));
6880 assert!(ambiguous.contains("host actor cancellation unavailable"));
6881 }
6882
6883 #[test]
6884 fn stale_fenced_accept_maps_to_typed_stale_member_rejection() {
6885 let expected = BridgeMemberIncarnation {
6886 mob_id: "mob".to_string(),
6887 agent_identity: "worker".to_string(),
6888 host_id: "host".to_string(),
6889 binding_generation: 1,
6890 member_session_id: "session-g1".to_string(),
6891 generation: 1,
6892 fence_token: 11,
6893 };
6894 let current = BridgeMemberIncarnation {
6895 binding_generation: 2,
6896 member_session_id: "session-g2".to_string(),
6897 generation: 2,
6898 fence_token: 12,
6899 ..expected.clone()
6900 };
6901 let cause = definite_no_effect_delivery_rejection(
6902 &crate::traits::RuntimeDriverError::StaleAuthority {
6903 reason: "G1 lost the residency cutover".to_string(),
6904 },
6905 Some(current.clone()),
6906 &expected,
6907 )
6908 .expect("stale authority is a definite no-effect rejection");
6909 assert_eq!(
6910 cause,
6911 BridgeDeliveryRejectionCause::StaleMemberResidency {
6912 expected,
6913 current: Some(current),
6914 }
6915 );
6916 }
6917
6918 #[test]
6919 fn stale_fenced_accept_without_replacement_never_fabricates_current_member() {
6920 let expected = BridgeMemberIncarnation {
6921 mob_id: "mob".to_string(),
6922 agent_identity: "worker".to_string(),
6923 host_id: "host".to_string(),
6924 binding_generation: 1,
6925 member_session_id: "session-g1".to_string(),
6926 generation: 1,
6927 fence_token: 11,
6928 };
6929 let cause = definite_no_effect_delivery_rejection(
6930 &crate::traits::RuntimeDriverError::StaleAuthority {
6931 reason: "G1 lost to vacate".to_string(),
6932 },
6933 None,
6934 &expected,
6935 )
6936 .expect("vacated stale authority remains a typed no-effect rejection");
6937 assert_eq!(
6938 cause,
6939 BridgeDeliveryRejectionCause::StaleMemberResidency {
6940 expected,
6941 current: None,
6942 }
6943 );
6944 }
6945
6946 #[test]
6947 fn member_events_reply_trims_outcomes_below_transport_budget() {
6948 let outcomes = (0..16)
6949 .map(|index| {
6950 meerkat_contracts::wire::supervisor_bridge::BridgeTurnOutcomeRecord {
6951 input_id: format!("input-{index}"),
6952 generation: 1,
6953 fence_token: 1,
6954 terminal_seq: index,
6955 outcome: meerkat_contracts::wire::supervisor_bridge::WireFlowTurnOutcome::RunFailed {
6956 detail: meerkat_contracts::wire::supervisor_bridge::WireFlowFailureDetail::complete(
6957 "x".repeat(60_000),
6958 ),
6959 },
6960 }
6961 })
6962 .collect();
6963 let reply = bounded_member_events_reply(MemberEventsWindow {
6964 runtime_incarnation: BridgeHostRuntimeIncarnation::new(),
6965 generation: 1,
6966 fence_token: 1,
6967 rows: Vec::new(),
6968 from_seq: 7,
6969 next_seq: 7,
6970 watermark: 6,
6971 turn_outcomes: outcomes,
6972 outcomes_complete: true,
6973 })
6974 .expect("bounded outcome page");
6975 let encoded = serde_json::to_vec(&reply).expect("serialize bounded reply");
6976 assert!(encoded.len() <= MEMBER_EVENTS_REPLY_MAX_BYTES);
6977 let BridgeReply::MemberEventsPage(page) = reply else {
6978 panic!("expected member events page")
6979 };
6980 assert_eq!(
6981 page.from_seq, 7,
6982 "the wire page preserves the host-resolved read floor even when no event row fits"
6983 );
6984 assert_eq!(page.next_seq, page.from_seq);
6985 assert!(page.turn_outcomes.len() < 16);
6986 assert!(!page.outcomes_complete);
6987 }
6988
6989 #[test]
6990 fn member_events_reply_rejects_one_oversized_event_without_cursor_skip() {
6991 let session_id =
6992 SessionId::parse("0195e9a0-0000-7000-8000-000000000001").expect("session id");
6993 let envelope = meerkat_core::event::EventEnvelope::new(
6994 "worker",
6995 1,
6996 Some("mob-size".to_string()),
6997 AgentEvent::ExtractionFailed {
6998 session_id,
6999 last_output: "x".repeat(MEMBER_EVENTS_REPLY_MAX_BYTES + 1024),
7000 attempts: 1,
7001 reason: "oversized".to_string(),
7002 },
7003 );
7004 let error = bounded_member_events_reply(MemberEventsWindow {
7005 runtime_incarnation: BridgeHostRuntimeIncarnation::new(),
7006 generation: 1,
7007 fence_token: 1,
7008 rows: vec![(7, envelope)],
7009 from_seq: 7,
7010 next_seq: 8,
7011 watermark: 7,
7012 turn_outcomes: Vec::new(),
7013 outcomes_complete: true,
7014 })
7015 .expect_err("single oversized event must reject instead of replay-looping empty pages");
7016 assert!(matches!(
7017 error,
7018 MemberEventsReplyError::OversizedEvent {
7019 generation: 1,
7020 durable_seq: 7,
7021 ..
7022 }
7023 ));
7024 }
7025
7026 #[test]
7027 fn member_events_reply_rejects_exhausted_row_without_max_replay_or_zero_wrap() {
7028 let envelope = meerkat_core::event::EventEnvelope::new(
7029 "worker",
7030 1,
7031 Some("mob-exhausted".to_string()),
7032 AgentEvent::TurnStarted { turn_number: 1 },
7033 );
7034 let error = bounded_member_events_reply(MemberEventsWindow {
7035 runtime_incarnation: BridgeHostRuntimeIncarnation::new(),
7036 generation: 1,
7037 fence_token: 1,
7038 rows: vec![(u64::MAX, envelope)],
7039 from_seq: u64::MAX,
7040 next_seq: u64::MAX,
7041 watermark: u64::MAX,
7042 turn_outcomes: Vec::new(),
7043 outcomes_complete: true,
7044 })
7045 .expect_err("a MAX row has no representable successor cursor");
7046 assert!(matches!(
7047 error,
7048 MemberEventsReplyError::Internal(reason)
7049 if reason.contains("sequence space is exhausted")
7050 ));
7051 }
7052
7053 #[test]
7054 fn member_events_reply_rejects_empty_exhausted_frontier_instead_of_spinning() {
7055 let error = bounded_member_events_reply(MemberEventsWindow {
7056 runtime_incarnation: BridgeHostRuntimeIncarnation::new(),
7057 generation: 1,
7058 fence_token: 1,
7059 rows: Vec::new(),
7060 from_seq: u64::MAX,
7061 next_seq: u64::MAX,
7062 watermark: u64::MAX,
7063 turn_outcomes: Vec::new(),
7064 outcomes_complete: true,
7065 })
7066 .expect_err("an empty MAX frontier cannot make progress");
7067 assert!(matches!(
7068 error,
7069 MemberEventsReplyError::Internal(reason)
7070 if reason.contains("empty MAX frontier")
7071 ));
7072 }
7073
7074 fn member_history_window(
7075 offset: usize,
7076 message_count: usize,
7077 rows: impl IntoIterator<Item = String>,
7078 ) -> MemberHistoryWindow {
7079 let messages: Vec<_> = rows
7080 .into_iter()
7081 .map(|text| {
7082 meerkat_core::types::Message::User(meerkat_core::types::UserMessage::text(text))
7083 })
7084 .collect();
7085 MemberHistoryWindow {
7086 generation: 4,
7087 page: meerkat_core::service::SessionHistoryPage {
7088 session_id: SessionId::parse("0195e9a0-0000-7000-8000-000000000001")
7089 .expect("session id"),
7090 message_count,
7091 offset,
7092 limit: Some(messages.len()),
7093 has_more: offset + messages.len() < message_count,
7094 messages,
7095 },
7096 }
7097 }
7098
7099 #[test]
7100 fn member_history_reply_returns_maximal_transport_safe_prefix() {
7101 let reply = bounded_member_history_reply(member_history_window(
7102 7,
7103 10,
7104 [
7105 "a".repeat(200_000),
7106 "b".repeat(200_000),
7107 "c".repeat(200_000),
7108 ],
7109 ))
7110 .expect("bounded history page");
7111 let encoded = serde_json::to_vec(&reply).expect("serialize bounded history reply");
7112 assert!(encoded.len() <= MEMBER_HISTORY_REPLY_MAX_BYTES);
7113 let BridgeReply::MemberHistoryPage(page) = reply else {
7114 panic!("expected member history page")
7115 };
7116 assert_eq!(page.page.from_index, 7);
7117 assert_eq!(page.page.messages.len(), 2);
7118 assert_eq!(page.page.next_index, Some(9));
7119 assert!(!page.page.complete);
7120 assert_eq!(page.page.message_count, 10);
7121 }
7122
7123 #[test]
7124 fn member_history_reply_rejects_single_poison_row_at_exact_index() {
7125 let error = bounded_member_history_reply(member_history_window(
7126 9,
7127 10,
7128 ["x".repeat(MEMBER_HISTORY_REPLY_MAX_BYTES + 64 * 1024)],
7129 ))
7130 .expect_err("one oversized history row must fail typed");
7131 assert!(matches!(
7132 error,
7133 MemberHistoryReplyError::RowTooLarge { index: 9, .. }
7134 ));
7135 }
7136
7137 #[test]
7138 fn member_history_reply_rejects_non_advancing_domain_page() {
7139 let error = bounded_member_history_reply(MemberHistoryWindow {
7140 generation: 4,
7141 page: meerkat_core::service::SessionHistoryPage {
7142 session_id: SessionId::parse("0195e9a0-0000-7000-8000-000000000001")
7143 .expect("session id"),
7144 message_count: 1,
7145 offset: 0,
7146 limit: Some(1),
7147 has_more: true,
7148 messages: Vec::new(),
7149 },
7150 })
7151 .expect_err("a non-advancing page must fail before it reaches the bridge");
7152 assert!(matches!(
7153 error,
7154 MemberHistoryReplyError::Internal(reason)
7155 if reason.contains("page projection failed") && reason.contains("serves none")
7156 ));
7157 }
7158
7159 const PEER_ID_RECEIVER: &str = "0b86269a-ed69-5e31-a1ce-9552506eebe3"; const PEER_ID_SUPERVISOR: &str = "0adb7b34-edb5-5898-bab4-3f7cb10ba4da"; const PEER_ID_OLD_SUPERVISOR: &str = "d15ad17e-57de-58da-a91a-466c87a3e0c4"; fn test_pubkey_bytes_for_peer_id(peer_id: &str) -> Option<[u8; 32]> {
7174 match peer_id {
7175 PEER_ID_RECEIVER => Some([0xaa; 32]),
7176 PEER_ID_SUPERVISOR => Some([0xbb; 32]),
7177 PEER_ID_OLD_SUPERVISOR => Some([0xdd; 32]),
7178 _ => None,
7179 }
7180 }
7181
7182 fn test_public_key_for_peer_id(peer_id: &str) -> Option<String> {
7183 test_pubkey_bytes_for_peer_id(peer_id)
7184 .map(|bytes| meerkat_comms::PubKey::new(bytes).to_pubkey_string())
7185 }
7186
7187 fn test_comms_name_from_address(address: &str) -> String {
7188 address
7189 .strip_prefix("inproc://")
7190 .unwrap_or(address)
7191 .to_string()
7192 }
7193
7194 fn test_projection_trust_dsl(
7195 runtime: &meerkat_comms::CommsRuntime,
7196 ) -> Arc<crate::handles::HandleDslAuthority> {
7197 let dsl = Arc::new(crate::handles::HandleDslAuthority::ephemeral());
7198 dsl.apply_signal(
7199 crate::meerkat_machine::dsl::MeerkatMachineSignal::Initialize,
7200 "test_projection_trust_initialize",
7201 )
7202 .expect("Initialize signal");
7203 dsl.apply_input(
7204 crate::meerkat_machine::dsl::MeerkatMachineInput::RegisterSession {
7205 session_id: crate::meerkat_machine::dsl::SessionId::from(
7206 "comms-drain-test-projection-trust",
7207 ),
7208 },
7209 "test_projection_trust_register",
7210 )
7211 .expect("RegisterSession input");
7212 crate::handles::RuntimePeerCommsHandle::install_generated_on(Arc::clone(&dsl), runtime)
7213 .expect("install generated peer-comms handle");
7214 dsl
7215 }
7216
7217 async fn add_test_projection_trust(
7218 runtime: &meerkat_comms::CommsRuntime,
7219 peer: TrustedPeerDescriptor,
7220 context: &str,
7221 ) {
7222 let dsl = test_projection_trust_dsl(runtime);
7223 add_test_projection_trust_with_dsl(runtime, dsl, peer, 1, context).await;
7224 }
7225
7226 async fn add_test_projection_trust_with_dsl(
7227 runtime: &meerkat_comms::CommsRuntime,
7228 dsl: Arc<crate::handles::HandleDslAuthority>,
7229 peer: TrustedPeerDescriptor,
7230 epoch: u64,
7231 context: &str,
7232 ) {
7233 let local_peer_id = runtime
7234 .peer_id()
7235 .unwrap_or_else(|| panic!("{context}: runtime peer_id unavailable"));
7236 let endpoint = crate::meerkat_machine::dsl::PeerEndpoint::from(&peer);
7237 dsl.apply_input(
7238 crate::meerkat_machine::dsl::MeerkatMachineInput::PublishLocalEndpoint {
7239 endpoint: crate::meerkat_machine::dsl::PeerEndpoint::new(
7240 "local",
7241 local_peer_id.to_string(),
7242 "inproc://local",
7243 [0x7f; 32],
7244 ),
7245 },
7246 "test_projection_trust_publish_local_endpoint",
7247 )
7248 .expect("PublishLocalEndpoint input");
7249 let transition = dsl
7250 .apply_input_with_transition(
7251 crate::meerkat_machine::dsl::MeerkatMachineInput::ApplyMobPeerOverlay {
7252 epoch,
7253 endpoints: std::collections::BTreeSet::from([endpoint.clone()]),
7254 },
7255 "test_projection_trust_apply_overlay",
7256 )
7257 .expect("ApplyMobPeerOverlay input");
7258 let obligation = crate::protocol_comms_trust_reconcile::extract_obligations_with_freshness(
7259 &transition,
7260 dsl.peer_projection_freshness_authority(),
7261 )
7262 .into_iter()
7263 .next()
7264 .expect("generated reconcile obligation");
7265 let authority =
7266 crate::protocol_comms_trust_reconcile::authority_for_endpoint(&obligation, &endpoint)
7267 .expect("generated authority");
7268 match runtime
7269 .apply_trust_mutation(CommsTrustMutation::AddTrustedPeer { peer, authority })
7270 .await
7271 .unwrap_or_else(|error| panic!("{context}: {error}"))
7272 {
7273 CommsTrustMutationResult::Added { .. } => {}
7274 other => panic!("{context}: unexpected trust mutation result {other:?}"),
7275 }
7276 }
7277
7278 async fn add_member_private_supervisor_trust_via_adapter(
7289 adapter: &Arc<MeerkatMachine>,
7290 session_id: &SessionId,
7291 runtime: &Arc<meerkat_comms::CommsRuntime>,
7292 supervisor: &TrustedPeerDescriptor,
7293 _epoch: u64,
7294 context: &str,
7295 ) {
7296 let runtime_dyn: Arc<dyn CommsRuntime> = Arc::clone(runtime) as Arc<dyn CommsRuntime>;
7297 adapter
7298 .stage_local_endpoint_for_comms_runtime(session_id, runtime_dyn.as_ref())
7299 .await
7300 .unwrap_or_else(|error| panic!("{context}: local endpoint rejected: {error}"));
7301 let obligation =
7302 supervisor_route_publish_obligation(adapter, session_id, supervisor, context)
7303 .await
7304 .unwrap_or_else(|error| panic!("{context}: {error}"));
7305 publish_supervisor_trust_from_generated_obligation(
7306 adapter,
7307 session_id,
7308 runtime_dyn.as_ref(),
7309 &obligation,
7310 )
7311 .await
7312 .unwrap_or_else(|error| panic!("{context}: private trust publish failed: {error}"));
7313 }
7314
7315 fn install_running_test_peer_comms_handle(
7316 runtime: &meerkat_comms::CommsRuntime,
7317 ) -> Arc<crate::handles::HandleDslAuthority> {
7318 let authority = Arc::new(crate::handles::HandleDslAuthority::ephemeral());
7319 let session_id = SessionId::new();
7320 authority
7321 .apply_signal(
7322 crate::meerkat_machine::dsl::MeerkatMachineSignal::Initialize,
7323 "running_test_peer_comms_handle_initialize",
7324 )
7325 .expect("test peer-comms authority should initialize");
7326 authority
7327 .apply_input(
7328 crate::meerkat_machine::dsl::MeerkatMachineInput::RegisterSession {
7329 session_id: crate::meerkat_machine::dsl::SessionId::from_domain(&session_id),
7330 },
7331 "running_test_peer_comms_handle_register",
7332 )
7333 .expect("test peer-comms authority should register a session");
7334 authority
7335 .apply_input(
7336 crate::meerkat_machine::dsl::MeerkatMachineInput::Prepare {
7337 session_id: crate::meerkat_machine::dsl::SessionId::from_domain(&session_id),
7338 run_id: crate::meerkat_machine::dsl::RunId::from_domain(
7339 &meerkat_core::lifecycle::RunId::new(),
7340 ),
7341 },
7342 "running_test_peer_comms_handle_prepare",
7343 )
7344 .expect("test peer-comms authority should enter Running");
7345 crate::handles::RuntimePeerCommsHandle::install_generated_on(
7346 Arc::clone(&authority),
7347 runtime,
7348 )
7349 .expect("install generated peer-comms handle");
7350 authority
7351 }
7352
7353 fn test_pubkey(seed: u8) -> meerkat_comms::PubKey {
7354 assert_ne!(seed, 0, "test pubkey seed must be non-zero");
7355 meerkat_comms::PubKey::new([seed; 32])
7356 }
7357
7358 fn test_supervisor_signing_public_key(seed: u8) -> String {
7359 encode_supervisor_signing_public_key([seed; 32])
7360 }
7361
7362 fn signing_public_key_for_descriptor(peer: &TrustedPeerDescriptor) -> String {
7363 encode_supervisor_signing_public_key(peer.pubkey)
7364 }
7365
7366 fn bridge_peer_spec_with_seed(name: &str, seed: u8, address: &str) -> BridgePeerSpec {
7367 let pubkey = test_pubkey(seed);
7368 BridgePeerSpec {
7369 name: name.to_string(),
7370 peer_id: pubkey.to_peer_id().as_str(),
7371 address: address.to_string(),
7372 pubkey: *pubkey.as_bytes(),
7373 }
7374 }
7375
7376 fn supervisor_bridge_spec() -> BridgePeerSpec {
7377 bridge_peer_spec_with_seed(
7378 "mob/__mob_supervisor__",
7379 0xbb,
7380 "inproc://mob/__mob_supervisor__",
7381 )
7382 }
7383
7384 fn current_supervisor_bridge_spec() -> BridgePeerSpec {
7385 bridge_peer_spec_with_seed(
7386 "mob/__mob_supervisor__",
7387 0xcc,
7388 "inproc://mob/__mob_supervisor__",
7389 )
7390 }
7391
7392 fn pubkey_sender_for_bridge_spec(spec: &BridgePeerSpec) -> String {
7393 meerkat_comms::PubKey::new(spec.pubkey).to_pubkey_string()
7394 }
7395
7396 fn old_supervisor_bridge_spec() -> BridgePeerSpec {
7397 bridge_peer_spec_with_seed(
7398 "mob/__mob_supervisor__",
7399 0xdd,
7400 "inproc://mob/__mob_supervisor__",
7401 )
7402 }
7403
7404 fn trusted_supervisor_descriptor(seed: u8) -> TrustedPeerDescriptor {
7405 TrustedPeerDescriptor::try_from(bridge_peer_spec_with_seed(
7406 "mob/__mob_supervisor__",
7407 seed,
7408 "inproc://mob/__mob_supervisor__",
7409 ))
7410 .expect("valid supervisor spec")
7411 }
7412
7413 fn trusted_peer_from_runtime(
7414 name: &str,
7415 runtime: &meerkat_comms::CommsRuntime,
7416 ) -> TrustedPeerDescriptor {
7417 let pubkey = runtime.public_key();
7418 TrustedPeerDescriptor::unsigned_with_pubkey(
7419 name,
7420 pubkey.to_peer_id().as_str(),
7421 *pubkey.as_bytes(),
7422 format!("inproc://{name}"),
7423 )
7424 .expect("valid non-zero trusted peer descriptor")
7425 }
7426
7427 async fn recorded_bridge_replies(
7428 sent_commands: &Arc<tokio::sync::Mutex<Vec<CommsCommand>>>,
7429 ) -> Vec<(meerkat_core::interaction::ResponseStatus, BridgeReply)> {
7430 sent_commands
7431 .lock()
7432 .await
7433 .iter()
7434 .filter_map(|cmd| match cmd {
7435 CommsCommand::PeerResponse { status, result, .. } => Some((
7436 *status,
7437 serde_json::from_value::<BridgeReply>(result.clone())
7438 .expect("recorded bridge response is typed"),
7439 )),
7440 _ => None,
7441 })
7442 .collect()
7443 }
7444
7445 fn assert_completed_bridge_ack(
7446 reply: &(meerkat_core::interaction::ResponseStatus, BridgeReply),
7447 ) {
7448 assert_eq!(
7449 reply.0,
7450 meerkat_core::interaction::ResponseStatus::Completed
7451 );
7452 assert!(
7453 matches!(reply.1, BridgeReply::Ack(BridgeAck { ok: true })),
7454 "expected completed bridge ack, got {:?}",
7455 reply.1
7456 );
7457 }
7458
7459 fn trusted_tcp_peer_from_runtime(
7460 name: &str,
7461 runtime: &meerkat_comms::CommsRuntime,
7462 ) -> TrustedPeerDescriptor {
7463 let pubkey = runtime.public_key();
7464 TrustedPeerDescriptor::unsigned_with_pubkey(
7465 name,
7466 pubkey.to_peer_id().as_str(),
7467 *pubkey.as_bytes(),
7468 runtime.advertised_address(),
7469 )
7470 .expect("valid non-zero TCP trusted peer descriptor")
7471 }
7472
7473 fn install_test_peer_request_response_authority(
7474 runtime: &Arc<meerkat_comms::CommsRuntime>,
7475 peer_handle: Arc<CountingPeerInteractionHandle>,
7476 ) {
7477 runtime.install_peer_request_response_authority(
7478 meerkat_comms::PeerRequestResponseAuthority::new(
7479 peer_handle,
7480 Arc::new(crate::handles::RuntimeInteractionStreamHandle::ephemeral()),
7481 ),
7482 );
7483 }
7484
7485 #[derive(Default)]
7486 struct CountingPeerInteractionHandle {
7487 inbound: std::sync::Mutex<HashSet<meerkat_core::PeerCorrelationId>>,
7488 inbound_modes: std::sync::Mutex<HashMap<meerkat_core::PeerCorrelationId, HandlingMode>>,
7489 request_received_count: std::sync::atomic::AtomicUsize,
7490 response_rejected_count: std::sync::atomic::AtomicUsize,
7491 response_replied_count: std::sync::atomic::AtomicUsize,
7492 reject_request_received: std::sync::atomic::AtomicBool,
7493 reject_response_progress: std::sync::atomic::AtomicBool,
7494 reject_response_terminal: std::sync::atomic::AtomicBool,
7495 }
7496
7497 impl CountingPeerInteractionHandle {
7498 fn rejecting_request_received() -> Self {
7499 let handle = Self::default();
7500 handle
7501 .reject_request_received
7502 .store(true, std::sync::atomic::Ordering::SeqCst);
7503 handle
7504 }
7505
7506 fn rejecting_response_progress() -> Self {
7507 let handle = Self::default();
7508 handle
7509 .reject_response_progress
7510 .store(true, std::sync::atomic::Ordering::SeqCst);
7511 handle
7512 }
7513
7514 fn rejecting_response_terminal() -> Self {
7515 let handle = Self::default();
7516 handle
7517 .reject_response_terminal
7518 .store(true, std::sync::atomic::Ordering::SeqCst);
7519 handle
7520 }
7521
7522 fn rejected(
7523 context: &'static str,
7524 corr_id: meerkat_core::PeerCorrelationId,
7525 ) -> meerkat_core::handles::DslTransitionError {
7526 meerkat_core::handles::DslTransitionError::guard_rejected(
7527 context,
7528 format!("test authority rejected corr_id {corr_id}"),
7529 )
7530 }
7531
7532 fn request_received_count(&self) -> usize {
7533 self.request_received_count
7534 .load(std::sync::atomic::Ordering::SeqCst)
7535 }
7536
7537 fn response_replied_count(&self) -> usize {
7538 self.response_replied_count
7539 .load(std::sync::atomic::Ordering::SeqCst)
7540 }
7541
7542 fn response_rejected_count(&self) -> usize {
7543 self.response_rejected_count
7544 .load(std::sync::atomic::Ordering::SeqCst)
7545 }
7546 }
7547
7548 impl meerkat_core::handles::PeerInteractionHandle for CountingPeerInteractionHandle {
7549 fn request_sent(
7550 &self,
7551 _corr_id: meerkat_core::PeerCorrelationId,
7552 ) -> Result<(), meerkat_core::handles::DslTransitionError> {
7553 Ok(())
7554 }
7555
7556 fn response_progress(
7557 &self,
7558 corr_id: meerkat_core::PeerCorrelationId,
7559 ) -> Result<(), meerkat_core::handles::DslTransitionError> {
7560 if self
7561 .reject_response_progress
7562 .load(std::sync::atomic::Ordering::SeqCst)
7563 {
7564 return Err(Self::rejected(
7565 "PeerInteractionHandle::response_progress",
7566 corr_id,
7567 ));
7568 }
7569 Ok(())
7570 }
7571
7572 fn response_terminal(
7573 &self,
7574 corr_id: meerkat_core::PeerCorrelationId,
7575 _disposition: meerkat_core::handles::PeerTerminalDisposition,
7576 ) -> Result<(), meerkat_core::handles::DslTransitionError> {
7577 if self
7578 .reject_response_terminal
7579 .load(std::sync::atomic::Ordering::SeqCst)
7580 {
7581 return Err(Self::rejected(
7582 "PeerInteractionHandle::response_terminal",
7583 corr_id,
7584 ));
7585 }
7586 Ok(())
7587 }
7588
7589 fn response_rejected(
7590 &self,
7591 _corr_id: meerkat_core::PeerCorrelationId,
7592 ) -> Result<(), meerkat_core::handles::DslTransitionError> {
7593 self.response_rejected_count
7594 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
7595 Ok(())
7596 }
7597
7598 fn request_timed_out(
7599 &self,
7600 _corr_id: meerkat_core::PeerCorrelationId,
7601 ) -> Result<(), meerkat_core::handles::DslTransitionError> {
7602 Ok(())
7603 }
7604
7605 fn request_send_failed(
7606 &self,
7607 _corr_id: meerkat_core::PeerCorrelationId,
7608 ) -> Result<(), meerkat_core::handles::DslTransitionError> {
7609 Ok(())
7610 }
7611
7612 fn classify_response_reply(
7613 &self,
7614 status: meerkat_core::ResponseStatus,
7615 ) -> Result<meerkat_core::TerminalityClass, meerkat_core::handles::DslTransitionError>
7616 {
7617 let generated = crate::handles::RuntimePeerInteractionHandle::ephemeral();
7618 meerkat_core::handles::PeerInteractionHandle::classify_response_reply(
7619 &generated, status,
7620 )
7621 }
7622
7623 fn request_received(
7624 &self,
7625 corr_id: meerkat_core::PeerCorrelationId,
7626 handling_mode: HandlingMode,
7627 ) -> Result<(), meerkat_core::handles::DslTransitionError> {
7628 if self
7629 .reject_request_received
7630 .load(std::sync::atomic::Ordering::SeqCst)
7631 {
7632 return Err(Self::rejected(
7633 "PeerInteractionHandle::request_received",
7634 corr_id,
7635 ));
7636 }
7637 let mut inbound = self.inbound.lock().expect("inbound mutex");
7638 if !inbound.insert(corr_id) {
7639 return Err(Self::rejected(
7640 "PeerInteractionHandle::request_received",
7641 corr_id,
7642 ));
7643 }
7644 self.inbound_modes
7645 .lock()
7646 .expect("inbound modes mutex")
7647 .insert(corr_id, handling_mode);
7648 self.request_received_count
7649 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
7650 Ok(())
7651 }
7652
7653 fn response_replied(
7654 &self,
7655 corr_id: meerkat_core::PeerCorrelationId,
7656 ) -> Result<(), meerkat_core::handles::DslTransitionError> {
7657 let mut inbound = self.inbound.lock().expect("inbound mutex");
7658 if !inbound.remove(&corr_id) {
7659 return Err(Self::rejected(
7660 "PeerInteractionHandle::response_replied",
7661 corr_id,
7662 ));
7663 }
7664 self.inbound_modes
7665 .lock()
7666 .expect("inbound modes mutex")
7667 .remove(&corr_id);
7668 self.response_replied_count
7669 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
7670 Ok(())
7671 }
7672
7673 fn outbound_state(
7674 &self,
7675 _corr_id: meerkat_core::PeerCorrelationId,
7676 ) -> Option<meerkat_core::OutboundPeerRequestState> {
7677 None
7678 }
7679
7680 fn inbound_state(
7681 &self,
7682 corr_id: meerkat_core::PeerCorrelationId,
7683 ) -> Option<meerkat_core::InboundPeerRequestState> {
7684 self.inbound
7685 .lock()
7686 .expect("inbound mutex")
7687 .contains(&corr_id)
7688 .then_some(meerkat_core::InboundPeerRequestState::Received)
7689 }
7690
7691 fn inbound_handling_mode(
7692 &self,
7693 corr_id: meerkat_core::PeerCorrelationId,
7694 ) -> Option<HandlingMode> {
7695 self.inbound_modes
7696 .lock()
7697 .expect("inbound modes mutex")
7698 .get(&corr_id)
7699 .copied()
7700 }
7701
7702 fn install_cleanup_observer(
7703 &self,
7704 _observer: Arc<dyn meerkat_core::handles::PeerInteractionCleanupObserver>,
7705 ) {
7706 }
7707 }
7708
7709 struct OneShotPeerRequestRuntime {
7710 notify: Arc<tokio::sync::Notify>,
7711 candidate: std::sync::Mutex<Option<PeerInputCandidate>>,
7712 peer_handle: Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>>,
7713 peer_request_response_handle: Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>>,
7714 completed_count: std::sync::atomic::AtomicUsize,
7715 abandonments:
7716 std::sync::Mutex<Vec<(InteractionId, meerkat_core::InteractionStreamAbandonReason)>>,
7717 }
7718
7719 impl OneShotPeerRequestRuntime {
7720 fn new(
7721 candidate: PeerInputCandidate,
7722 peer_handle: Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>>,
7723 ) -> Self {
7724 Self {
7725 notify: Arc::new(tokio::sync::Notify::new()),
7726 candidate: std::sync::Mutex::new(Some(candidate)),
7727 peer_handle,
7728 peer_request_response_handle: None,
7729 completed_count: std::sync::atomic::AtomicUsize::new(0),
7730 abandonments: std::sync::Mutex::new(Vec::new()),
7731 }
7732 }
7733
7734 fn with_complete_authority(
7735 candidate: PeerInputCandidate,
7736 peer_handle: Arc<dyn meerkat_core::handles::PeerInteractionHandle>,
7737 ) -> Self {
7738 Self {
7739 notify: Arc::new(tokio::sync::Notify::new()),
7740 candidate: std::sync::Mutex::new(Some(candidate)),
7741 peer_handle: Some(peer_handle.clone()),
7742 peer_request_response_handle: Some(peer_handle),
7743 completed_count: std::sync::atomic::AtomicUsize::new(0),
7744 abandonments: std::sync::Mutex::new(Vec::new()),
7745 }
7746 }
7747
7748 fn completed_count(&self) -> usize {
7749 self.completed_count
7750 .load(std::sync::atomic::Ordering::SeqCst)
7751 }
7752
7753 fn abandonment_reasons(&self) -> Vec<meerkat_core::InteractionStreamAbandonReason> {
7754 self.abandonments
7755 .lock()
7756 .expect("abandonments mutex")
7757 .iter()
7758 .map(|(_, reason)| *reason)
7759 .collect()
7760 }
7761 }
7762
7763 #[async_trait::async_trait]
7764 impl CommsRuntime for OneShotPeerRequestRuntime {
7765 async fn drain_messages(&self) -> Vec<String> {
7766 Vec::new()
7767 }
7768
7769 fn inbox_notify(&self) -> Arc<tokio::sync::Notify> {
7770 self.notify.clone()
7771 }
7772
7773 fn dismiss_received(&self) -> bool {
7774 self.candidate.lock().expect("candidate mutex").is_none()
7775 }
7776
7777 fn peer_interaction_handle(
7778 &self,
7779 ) -> Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>> {
7780 self.peer_handle.clone()
7781 }
7782
7783 fn peer_request_response_authority_handle(
7784 &self,
7785 ) -> Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>> {
7786 self.peer_request_response_handle.clone()
7787 }
7788
7789 async fn drain_classified_inbox_interactions(
7790 &self,
7791 ) -> Result<
7792 Vec<meerkat_core::interaction::ClassifiedInboxInteraction>,
7793 meerkat_core::agent::CommsCapabilityError,
7794 > {
7795 Ok(self
7796 .candidate
7797 .lock()
7798 .expect("candidate mutex")
7799 .take()
7800 .into_iter()
7801 .collect())
7802 }
7803
7804 fn mark_interaction_complete(&self, _id: &InteractionId) {
7805 self.completed_count
7806 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
7807 }
7808
7809 fn abandon_interaction_stream(
7810 &self,
7811 id: &InteractionId,
7812 reason: meerkat_core::InteractionStreamAbandonReason,
7813 ) {
7814 self.abandonments
7815 .lock()
7816 .expect("abandonments mutex")
7817 .push((*id, reason));
7818 }
7819 }
7820
7821 fn inbound_peer_request_candidate(class: PeerInputClass) -> PeerInputCandidate {
7822 assert!(
7823 matches!(
7824 class,
7825 PeerInputClass::SilentRequest | PeerInputClass::ActionableRequest
7826 ),
7827 "test helper only builds inbound peer requests"
7828 );
7829 let id = InteractionId(Uuid::new_v4());
7830 let intent = format!("test.inbound.{class:?}");
7831 let sender = "partial-authority-peer";
7832 PeerInputCandidate {
7833 interaction: InboxInteraction {
7834 objective_id: None,
7835 sender_taint: None,
7836 id,
7837 from_route: None,
7838 from: sender.to_string(),
7839 content: InteractionContent::Request {
7840 intent: intent.clone(),
7841 params: json!({ "ok": true }),
7842 blocks: None,
7843 },
7844 rendered_text: String::new(),
7845 handling_mode: HandlingMode::Queue,
7846 render_metadata: None,
7847 },
7848 ingress: PeerIngressFact::peer(
7849 id,
7850 class,
7851 meerkat_core::PeerIngressKind::Request,
7852 Some(meerkat_core::PeerIngressAuthDecision::Required),
7853 PeerIngressIdentity::new(
7854 PeerId::new(),
7855 sender,
7856 PeerIngressConvention::Request {
7857 request_id: id.to_string(),
7858 intent,
7859 },
7860 ),
7861 ),
7862 lifecycle_peer: None,
7863 response_terminality: None,
7864 }
7865 }
7866
7867 fn response_candidate(
7868 class: PeerInputClass,
7869 response_terminality: Option<meerkat_core::interaction::TerminalityClass>,
7870 ) -> PeerInputCandidate {
7871 assert!(
7872 matches!(
7873 class,
7874 PeerInputClass::ResponseProgress | PeerInputClass::ResponseTerminal
7875 ),
7876 "test helper only builds peer response candidates"
7877 );
7878 let id = InteractionId(Uuid::new_v4());
7879 let in_reply_to = InteractionId(Uuid::new_v4());
7880 PeerInputCandidate {
7881 interaction: InboxInteraction {
7882 objective_id: None,
7883 sender_taint: None,
7884 id,
7885 from_route: Some(PeerId::new()),
7886 from: "response-peer".to_string(),
7887 content: InteractionContent::Response {
7888 in_reply_to,
7889 status: meerkat_core::interaction::ResponseStatus::Completed,
7890 result: json!({ "ok": true }),
7891 blocks: None,
7892 },
7893 rendered_text: String::new(),
7894 handling_mode: HandlingMode::Queue,
7895 render_metadata: None,
7896 },
7897 ingress: PeerIngressFact::peer(
7898 id,
7899 class,
7900 meerkat_core::PeerIngressKind::Response,
7901 Some(meerkat_core::PeerIngressAuthDecision::Required),
7902 PeerIngressIdentity::new(
7903 PeerId::new(),
7904 "response-peer",
7905 PeerIngressConvention::Response {
7906 in_reply_to,
7907 status: meerkat_core::interaction::ResponseStatus::Completed,
7908 },
7909 ),
7910 ),
7911 lifecycle_peer: None,
7912 response_terminality,
7913 }
7914 }
7915
7916 struct ClassifiedDrainOutcomeRuntime {
7919 notify: Arc<tokio::sync::Notify>,
7920 fail_classified_drain: bool,
7921 }
7922
7923 impl ClassifiedDrainOutcomeRuntime {
7924 fn new(fail_classified_drain: bool) -> Self {
7925 Self {
7926 notify: Arc::new(tokio::sync::Notify::new()),
7927 fail_classified_drain,
7928 }
7929 }
7930 }
7931
7932 #[async_trait::async_trait]
7933 impl CommsRuntime for ClassifiedDrainOutcomeRuntime {
7934 async fn drain_messages(&self) -> Vec<String> {
7935 Vec::new()
7936 }
7937
7938 fn inbox_notify(&self) -> Arc<tokio::sync::Notify> {
7939 self.notify.clone()
7940 }
7941
7942 async fn drain_classified_inbox_interactions(
7943 &self,
7944 ) -> Result<
7945 Vec<meerkat_core::interaction::ClassifiedInboxInteraction>,
7946 meerkat_core::agent::CommsCapabilityError,
7947 > {
7948 if self.fail_classified_drain {
7949 Err(meerkat_core::agent::CommsCapabilityError::Unsupported(
7950 "synthetic classification fault".to_string(),
7951 ))
7952 } else {
7953 Ok(Vec::new())
7954 }
7955 }
7956 }
7957
7958 async fn spawn_authorized_test_comms_drain(
7959 adapter: Arc<MeerkatMachine>,
7960 session_id: SessionId,
7961 runtime: Arc<dyn CommsRuntime>,
7962 idle_timeout: Duration,
7963 ) -> crate::tokio::task::JoinHandle<()> {
7964 adapter
7970 .test_authorize_direct_comms_drain_runtime(&session_id, Arc::clone(&runtime))
7971 .await
7972 .expect("direct-drain test authority should accept");
7973 spawn_comms_drain(adapter, session_id, runtime, Some(idle_timeout))
7974 }
7975
7976 #[tokio::test]
7984 async fn classified_drain_error_fails_closed_not_idle_timeout() {
7985 let adapter = Arc::new(MeerkatMachine::ephemeral());
7986 let session_id = SessionId::new();
7987 adapter
7988 .register_session(session_id.clone())
7989 .await
7990 .expect("register session");
7991
7992 let runtime = Arc::new(ClassifiedDrainOutcomeRuntime::new(true));
7995 let drain = spawn_authorized_test_comms_drain(
7996 adapter.clone(),
7997 session_id.clone(),
7998 runtime.clone(),
7999 Duration::from_secs(60),
8000 )
8001 .await;
8002
8003 tokio::time::timeout(Duration::from_secs(2), drain)
8004 .await
8005 .expect(
8006 "a classified-drain error must fail closed promptly, NOT idle until the 60s timeout",
8007 )
8008 .expect("drain task should not panic");
8009 }
8010
8011 #[tokio::test]
8016 async fn empty_classified_drain_idles_rather_than_exiting() {
8017 let adapter = Arc::new(MeerkatMachine::ephemeral());
8018 let session_id = SessionId::new();
8019 adapter
8020 .register_session(session_id.clone())
8021 .await
8022 .expect("register session");
8023
8024 let runtime = Arc::new(ClassifiedDrainOutcomeRuntime::new(false));
8025 let drain = spawn_authorized_test_comms_drain(
8026 adapter.clone(),
8027 session_id.clone(),
8028 runtime.clone(),
8029 Duration::from_secs(60),
8030 )
8031 .await;
8032
8033 assert!(
8036 tokio::time::timeout(Duration::from_millis(200), drain)
8037 .await
8038 .is_err(),
8039 "an empty (Ok) inbox must idle, not exit promptly"
8040 );
8041 }
8042
8043 #[tokio::test]
8044 async fn response_without_machine_terminality_is_rejected_before_progress_or_admission() {
8045 let adapter = Arc::new(MeerkatMachine::ephemeral());
8046 let session_id = SessionId::new();
8047 adapter
8048 .register_session(session_id.clone())
8049 .await
8050 .expect("register session");
8051
8052 let runtime = Arc::new(OneShotPeerRequestRuntime::new(
8053 response_candidate(PeerInputClass::ResponseTerminal, None),
8054 None,
8055 ));
8056 let drain = spawn_authorized_test_comms_drain(
8057 adapter.clone(),
8058 session_id.clone(),
8059 runtime.clone(),
8060 Duration::from_millis(10),
8061 )
8062 .await;
8063
8064 tokio::time::timeout(Duration::from_secs(1), drain)
8065 .await
8066 .expect("drain should exit after one response candidate")
8067 .expect("drain task should not panic");
8068
8069 let snapshot = adapter
8070 .meerkat_machine_spine_snapshot(&session_id)
8071 .await
8072 .expect("registered session snapshot");
8073 assert_eq!(
8074 snapshot.ledger.input_count, 0,
8075 "missing terminality must fail closed before runtime admission"
8076 );
8077 }
8078
8079 #[tokio::test]
8080 async fn malformed_response_terminality_rejects_pending_peer_truth_via_authority() {
8081 let adapter = Arc::new(MeerkatMachine::ephemeral());
8082 let session_id = SessionId::new();
8083 adapter
8084 .register_session(session_id.clone())
8085 .await
8086 .expect("register session");
8087
8088 let peer_handle = Arc::new(CountingPeerInteractionHandle::default());
8089 let peer_authority: Arc<dyn meerkat_core::handles::PeerInteractionHandle> =
8090 peer_handle.clone();
8091 let runtime = Arc::new(OneShotPeerRequestRuntime::new(
8092 response_candidate(PeerInputClass::ResponseTerminal, None),
8093 Some(peer_authority),
8094 ));
8095 let drain = spawn_authorized_test_comms_drain(
8096 adapter.clone(),
8097 session_id.clone(),
8098 runtime.clone(),
8099 Duration::from_millis(10),
8100 )
8101 .await;
8102
8103 tokio::time::timeout(Duration::from_secs(1), drain)
8104 .await
8105 .expect("drain should exit after one malformed response candidate")
8106 .expect("drain task should not panic");
8107
8108 assert_eq!(
8109 peer_handle.response_rejected_count(),
8110 1,
8111 "malformed response observations must terminalize pending peer truth through generated authority"
8112 );
8113 let snapshot = adapter
8114 .meerkat_machine_spine_snapshot(&session_id)
8115 .await
8116 .expect("registered session snapshot");
8117 assert_eq!(
8118 snapshot.ledger.input_count, 0,
8119 "malformed response terminality must fail closed before runtime admission"
8120 );
8121 }
8122
8123 #[tokio::test]
8124 async fn rejected_peer_response_transition_fails_closed_before_machine_admission() {
8125 for (candidate, peer_handle) in [
8126 (
8127 response_candidate(
8128 PeerInputClass::ResponseTerminal,
8129 Some(meerkat_core::interaction::TerminalityClass::Terminal {
8130 disposition: meerkat_core::interaction::TerminalDisposition::Completed,
8131 }),
8132 ),
8133 Arc::new(CountingPeerInteractionHandle::rejecting_response_terminal()),
8134 ),
8135 (
8136 response_candidate(
8137 PeerInputClass::ResponseProgress,
8138 Some(meerkat_core::interaction::TerminalityClass::Progress),
8139 ),
8140 Arc::new(CountingPeerInteractionHandle::rejecting_response_progress()),
8141 ),
8142 ] {
8143 let adapter = Arc::new(MeerkatMachine::ephemeral());
8144 let session_id = SessionId::new();
8145 adapter
8146 .register_session(session_id.clone())
8147 .await
8148 .expect("register session");
8149
8150 let runtime = Arc::new(OneShotPeerRequestRuntime::new(
8151 candidate,
8152 Some(peer_handle.clone()),
8153 ));
8154 let drain = spawn_authorized_test_comms_drain(
8155 adapter.clone(),
8156 session_id.clone(),
8157 runtime.clone(),
8158 Duration::from_millis(10),
8159 )
8160 .await;
8161
8162 tokio::time::timeout(Duration::from_secs(1), drain)
8163 .await
8164 .expect("drain should exit after one rejected response candidate")
8165 .expect("drain task should not panic");
8166
8167 assert_eq!(
8168 peer_handle.response_rejected_count(),
8169 1,
8170 "rejected generated peer-response transition should terminalize peer truth through response rejection"
8171 );
8172 let snapshot = adapter
8173 .meerkat_machine_spine_snapshot(&session_id)
8174 .await
8175 .expect("registered session snapshot");
8176 assert_eq!(
8177 snapshot.ledger.input_count, 0,
8178 "rejected generated peer-response transition must fail closed before runtime admission"
8179 );
8180 }
8181 }
8182
8183 #[tokio::test]
8184 async fn rejected_peer_request_recording_rejects_inbound_request_before_machine_admission() {
8185 for class in [
8186 PeerInputClass::SilentRequest,
8187 PeerInputClass::ActionableRequest,
8188 ] {
8189 let adapter = Arc::new(MeerkatMachine::ephemeral());
8190 let session_id = SessionId::new();
8191 adapter
8192 .register_session(session_id.clone())
8193 .await
8194 .expect("register session");
8195
8196 let peer_handle = Arc::new(CountingPeerInteractionHandle::rejecting_request_received());
8197 let peer_authority: Arc<dyn meerkat_core::handles::PeerInteractionHandle> = peer_handle;
8198 let runtime = Arc::new(OneShotPeerRequestRuntime::with_complete_authority(
8199 inbound_peer_request_candidate(class),
8200 peer_authority,
8201 ));
8202 let drain = spawn_authorized_test_comms_drain(
8203 adapter.clone(),
8204 session_id.clone(),
8205 runtime.clone(),
8206 Duration::from_millis(10),
8207 )
8208 .await;
8209
8210 tokio::time::timeout(Duration::from_secs(1), drain)
8211 .await
8212 .expect("drain should exit after one candidate")
8213 .expect("drain task should not panic");
8214
8215 let snapshot = adapter
8216 .meerkat_machine_spine_snapshot(&session_id)
8217 .await
8218 .expect("registered session snapshot");
8219 assert_eq!(
8220 snapshot.ledger.input_count, 0,
8221 "rejected PeerRequestReceived must not admit {class:?} into machine work"
8222 );
8223 assert_eq!(runtime.completed_count(), 0);
8224 assert_eq!(
8225 runtime.abandonment_reasons(),
8226 vec![meerkat_core::InteractionStreamAbandonReason::AdmissionRejected],
8227 "rejected {class:?} must use the typed abandonment path"
8228 );
8229 }
8230 }
8231
8232 #[tokio::test]
8233 async fn incomplete_peer_request_authority_rejects_inbound_request_before_machine_admission() {
8234 for class in [
8235 PeerInputClass::SilentRequest,
8236 PeerInputClass::ActionableRequest,
8237 ] {
8238 let adapter = Arc::new(MeerkatMachine::ephemeral());
8239 let session_id = SessionId::new();
8240 adapter
8241 .register_session(session_id.clone())
8242 .await
8243 .expect("register session");
8244
8245 let peer_handle = Arc::new(CountingPeerInteractionHandle::default());
8246 let peer_authority: Arc<dyn meerkat_core::handles::PeerInteractionHandle> =
8247 peer_handle.clone();
8248 let runtime = Arc::new(OneShotPeerRequestRuntime::new(
8249 inbound_peer_request_candidate(class),
8250 Some(peer_authority),
8251 ));
8252 let drain = spawn_authorized_test_comms_drain(
8253 adapter.clone(),
8254 session_id.clone(),
8255 runtime.clone(),
8256 Duration::from_millis(10),
8257 )
8258 .await;
8259
8260 tokio::time::timeout(Duration::from_secs(1), drain)
8261 .await
8262 .expect("drain should exit after one candidate")
8263 .expect("drain task should not panic");
8264
8265 let snapshot = adapter
8266 .meerkat_machine_spine_snapshot(&session_id)
8267 .await
8268 .expect("registered session snapshot");
8269 assert_eq!(
8270 peer_handle.request_received_count(),
8271 0,
8272 "partial authority must not fire PeerRequestReceived for {class:?}"
8273 );
8274 assert_eq!(
8275 snapshot.ledger.input_count, 0,
8276 "partial authority must not admit {class:?} into machine work"
8277 );
8278 assert_eq!(runtime.completed_count(), 0);
8279 assert_eq!(
8280 runtime.abandonment_reasons(),
8281 vec![meerkat_core::InteractionStreamAbandonReason::AdmissionRejected],
8282 "rejected {class:?} must use the typed abandonment path"
8283 );
8284 }
8285 }
8286
8287 #[tokio::test]
8288 async fn absent_peer_request_authority_rejects_inbound_request_before_machine_admission() {
8289 for class in [
8290 PeerInputClass::SilentRequest,
8291 PeerInputClass::ActionableRequest,
8292 ] {
8293 let adapter = Arc::new(MeerkatMachine::ephemeral());
8294 let session_id = SessionId::new();
8295 adapter
8296 .register_session(session_id.clone())
8297 .await
8298 .expect("register session");
8299
8300 let runtime = Arc::new(OneShotPeerRequestRuntime::new(
8301 inbound_peer_request_candidate(class),
8302 None,
8303 ));
8304 let drain = spawn_authorized_test_comms_drain(
8305 adapter.clone(),
8306 session_id.clone(),
8307 runtime.clone(),
8308 Duration::from_millis(10),
8309 )
8310 .await;
8311
8312 tokio::time::timeout(Duration::from_secs(1), drain)
8313 .await
8314 .expect("drain should exit after one candidate")
8315 .expect("drain task should not panic");
8316
8317 let snapshot = adapter
8318 .meerkat_machine_spine_snapshot(&session_id)
8319 .await
8320 .expect("registered session snapshot");
8321 assert_eq!(
8322 snapshot.ledger.input_count, 0,
8323 "missing authority must not admit {class:?} into machine work"
8324 );
8325 assert_eq!(runtime.completed_count(), 0);
8326 assert_eq!(
8327 runtime.abandonment_reasons(),
8328 vec![meerkat_core::InteractionStreamAbandonReason::AdmissionRejected],
8329 "rejected {class:?} must use the typed abandonment path"
8330 );
8331 }
8332 }
8333
8334 fn bridge_sender_fact(sender: &str) -> PeerIngressFact {
8335 let id = InteractionId(Uuid::new_v4());
8336 bridge_sender_fact_with_id(id, sender)
8337 }
8338
8339 fn bridge_sender_fact_with_display(
8340 canonical_peer_id: PeerId,
8341 display_label: &str,
8342 ) -> PeerIngressFact {
8343 let id = InteractionId(Uuid::new_v4());
8344 PeerIngressFact::peer(
8345 id,
8346 PeerInputClass::ActionableRequest,
8347 meerkat_core::PeerIngressKind::Request,
8348 Some(meerkat_core::PeerIngressAuthDecision::Exempt(
8349 meerkat_core::PeerIngressAuthExemption::SupervisorBridge,
8350 )),
8351 PeerIngressIdentity::new(
8352 canonical_peer_id,
8353 display_label,
8354 meerkat_core::PeerIngressConvention::Request {
8355 request_id: id.to_string(),
8356 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
8357 },
8358 ),
8359 )
8360 }
8361
8362 fn bridge_sender_fact_with_id(id: InteractionId, sender: &str) -> PeerIngressFact {
8363 if let Ok(pubkey) = meerkat_comms::PubKey::from_pubkey_string(sender) {
8364 return PeerIngressFact::peer(
8365 id,
8366 PeerInputClass::ActionableRequest,
8367 meerkat_core::PeerIngressKind::Request,
8368 Some(meerkat_core::PeerIngressAuthDecision::Exempt(
8369 meerkat_core::PeerIngressAuthExemption::SupervisorBridge,
8370 )),
8371 PeerIngressIdentity::new(
8372 pubkey.to_peer_id(),
8373 sender,
8374 meerkat_core::PeerIngressConvention::Request {
8375 request_id: id.to_string(),
8376 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
8377 },
8378 )
8379 .with_signing_pubkey(*pubkey.as_bytes()),
8380 );
8381 }
8382 if let Ok(peer_id) = PeerId::parse(sender) {
8383 return PeerIngressFact::peer(
8384 id,
8385 PeerInputClass::ActionableRequest,
8386 meerkat_core::PeerIngressKind::Request,
8387 Some(meerkat_core::PeerIngressAuthDecision::Exempt(
8388 meerkat_core::PeerIngressAuthExemption::SupervisorBridge,
8389 )),
8390 PeerIngressIdentity::new(
8391 peer_id,
8392 sender,
8393 meerkat_core::PeerIngressConvention::Request {
8394 request_id: id.to_string(),
8395 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
8396 },
8397 ),
8398 );
8399 }
8400 PeerIngressFact::peer(
8401 id,
8402 PeerInputClass::ActionableRequest,
8403 meerkat_core::PeerIngressKind::Request,
8404 Some(meerkat_core::PeerIngressAuthDecision::Exempt(
8405 meerkat_core::PeerIngressAuthExemption::SupervisorBridge,
8406 )),
8407 PeerIngressIdentity::new(
8408 PeerId::new(),
8409 sender,
8410 meerkat_core::PeerIngressConvention::Request {
8411 request_id: id.to_string(),
8412 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
8413 },
8414 ),
8415 )
8416 }
8417
8418 #[test]
8419 fn generated_sender_peer_id_uses_canonical_peer_id_not_display_name() {
8420 let current_peer_id = PeerId::parse(PEER_ID_SUPERVISOR).expect("valid supervisor peer id");
8421 let attacker_peer_id =
8422 PeerId::parse(PEER_ID_OLD_SUPERVISOR).expect("valid attacker peer id");
8423 let sender = bridge_sender_fact_with_display(attacker_peer_id, "mob/__mob_supervisor__");
8424
8425 assert_ne!(
8426 generated_sender_peer_id(&sender),
8427 Some(current_peer_id.as_str()),
8428 "display label must not become generated admission identity"
8429 );
8430 }
8431
8432 #[test]
8433 fn sender_matches_bridge_peer_rejects_same_display_name_with_different_peer_id() {
8434 let attacker_peer_id =
8435 PeerId::parse(PEER_ID_OLD_SUPERVISOR).expect("valid attacker peer id");
8436 let peer = bridge_peer_identity(&supervisor_bridge_spec(), "test")
8437 .expect("valid bridge peer identity");
8438 let sender = bridge_sender_fact_with_display(attacker_peer_id, peer.name.as_str());
8439
8440 assert!(
8441 !sender_matches_bridge_peer(&sender, &peer),
8442 "display label must not prove bridge peer authority"
8443 );
8444 }
8445
8446 #[tokio::test]
8447 async fn bridge_response_route_requires_known_canonical_peer_id() {
8448 let peer = meerkat_comms::Keypair::generate();
8449 let peer_pubkey = peer.public_key();
8450 let peer_spec = TrustedPeerDescriptor::unsigned_with_pubkey(
8451 "bridge-response-peer",
8452 peer_pubkey.to_peer_id().as_str(),
8453 *peer_pubkey.as_bytes(),
8454 "inproc://bridge-response-peer",
8455 )
8456 .expect("valid peer spec");
8457 let runtime = Arc::new(
8458 meerkat_comms::CommsRuntime::inproc_only("bridge-response-route").expect("runtime"),
8459 );
8460 let runtime_dyn: Arc<dyn CommsRuntime> = runtime.clone();
8461 add_test_projection_trust(runtime.as_ref(), peer_spec.clone(), "trust peer").await;
8462
8463 let route = resolve_peer_route(&runtime_dyn, peer_spec.peer_id)
8464 .await
8465 .expect("canonical peer id should resolve through the peer directory");
8466
8467 assert_eq!(route.peer_id, peer_spec.peer_id);
8468 let unknown_peer_id = PeerId::new();
8469 assert!(
8470 resolve_peer_route(&runtime_dyn, unknown_peer_id)
8471 .await
8472 .is_none(),
8473 "unknown PeerIds must not synthesize bridge response routes"
8474 );
8475 }
8476
8477 #[tokio::test]
8478 async fn bridge_response_route_uses_pubkey_sender_not_spoofed_payload_peer_id() {
8479 let runtime = Arc::new(
8480 meerkat_comms::CommsRuntime::inproc_only("bridge-response-spoof").expect("runtime"),
8481 );
8482 let runtime_dyn: Arc<dyn CommsRuntime> = runtime.clone();
8483 let spoofed_key = meerkat_comms::Keypair::generate();
8484 let spoofed_pubkey = spoofed_key.public_key();
8485 let spoofed_peer_id = spoofed_pubkey.to_peer_id();
8486 let spoofed_target = TrustedPeerDescriptor::unsigned_with_pubkey(
8487 "spoofed-target",
8488 spoofed_peer_id.as_str(),
8489 *spoofed_pubkey.as_bytes(),
8490 "inproc://spoofed-target",
8491 )
8492 .expect("valid spoofed target");
8493 add_test_projection_trust(runtime.as_ref(), spoofed_target, "trust spoofed target").await;
8494 let sender_key = meerkat_comms::Keypair::generate();
8495 let sender_pubkey = sender_key.public_key();
8496 let spoofed_peer_id = spoofed_peer_id.as_str();
8497 let command = BridgeCommand::BindMember(
8498 meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
8499 supervisor: BridgePeerSpec {
8500 name: "mob/__mob_supervisor__".to_string(),
8501 peer_id: spoofed_peer_id.clone(),
8502 address: "inproc://mob/__mob_supervisor__".to_string(),
8503 pubkey: *sender_pubkey.as_bytes(),
8504 },
8505 epoch: 1,
8506 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
8507 expected_peer_id: PEER_ID_RECEIVER.to_string(),
8508 expected_address: "inproc://receiver".to_string(),
8509 bootstrap_token: "bootstrap".to_string().into(),
8510 },
8511 );
8512 let id = InteractionId(Uuid::new_v4());
8513 let sender_label = sender_pubkey.to_pubkey_string();
8514 let candidate = PeerInputCandidate {
8515 interaction: InboxInteraction {
8516 objective_id: None,
8517 sender_taint: None,
8518 id,
8519 from_route: None,
8520 from: sender_pubkey.to_pubkey_string(),
8521 content: InteractionContent::Request {
8522 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
8523 params: serde_json::to_value(command).expect("serialize bridge command"),
8524 blocks: None,
8525 },
8526 rendered_text: String::new(),
8527 handling_mode: HandlingMode::Queue,
8528 render_metadata: None,
8529 },
8530 ingress: bridge_sender_fact_with_id(id, &sender_label),
8531 lifecycle_peer: None,
8532 response_terminality: None,
8533 };
8534
8535 let route = resolve_bridge_response_route(&runtime_dyn, &candidate)
8536 .await
8537 .expect("raw pubkey sender should resolve to its derived route");
8538
8539 assert_eq!(route.peer_id, sender_pubkey.to_peer_id());
8540 assert_ne!(
8541 route.peer_id.as_str(),
8542 spoofed_peer_id,
8543 "bridge replies must not route to the caller-supplied supervisor.peer_id"
8544 );
8545 }
8546
8547 #[tokio::test]
8548 async fn bridge_response_route_accepts_trusted_display_name_sender() {
8549 let peer = meerkat_comms::Keypair::generate();
8550 let peer_pubkey = peer.public_key();
8551 let peer_spec = TrustedPeerDescriptor::unsigned_with_pubkey(
8552 "mob/__mob_supervisor__",
8553 peer_pubkey.to_peer_id().as_str(),
8554 *peer_pubkey.as_bytes(),
8555 "inproc://mob/__mob_supervisor__",
8556 )
8557 .expect("valid peer spec");
8558 let runtime = Arc::new(
8559 meerkat_comms::CommsRuntime::inproc_only("bridge-response-display").expect("runtime"),
8560 );
8561 let runtime_dyn: Arc<dyn CommsRuntime> = runtime.clone();
8562 add_test_projection_trust(runtime.as_ref(), peer_spec.clone(), "trust peer").await;
8563
8564 let command = BridgeCommand::ObserveMember(BridgeSupervisorPayload {
8565 supervisor: BridgePeerSpec::from(peer_spec.clone()),
8566 epoch: 1,
8567 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
8568 });
8569 let id = InteractionId(Uuid::new_v4());
8570 let ingress = PeerIngressFact::peer(
8571 id,
8572 PeerInputClass::ActionableRequest,
8573 meerkat_core::PeerIngressKind::Request,
8574 Some(meerkat_core::PeerIngressAuthDecision::Exempt(
8575 meerkat_core::PeerIngressAuthExemption::SupervisorBridge,
8576 )),
8577 PeerIngressIdentity::new(
8578 peer_pubkey.to_peer_id(),
8579 peer_spec.name.as_str(),
8580 meerkat_core::PeerIngressConvention::Request {
8581 request_id: id.to_string(),
8582 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
8583 },
8584 )
8585 .with_signing_pubkey(*peer_pubkey.as_bytes()),
8586 );
8587 let candidate = bridge_candidate_with_ingress(peer_spec.name.as_str(), &command, ingress);
8588 let route = resolve_bridge_response_route(&runtime_dyn, &candidate)
8589 .await
8590 .expect("trusted display-name sender should resolve through peer directory");
8591
8592 assert_eq!(route.peer_id, peer_spec.peer_id);
8593 assert_eq!(
8594 route.display_name.as_ref().map(|name| name.as_str()),
8595 Some(peer_spec.name.as_str())
8596 );
8597 }
8598
8599 #[tokio::test]
8600 async fn bridge_response_route_uses_typed_sender_peer_id_not_display_or_payload_identity() {
8601 let peer = meerkat_comms::Keypair::generate();
8602 let peer_pubkey = peer.public_key();
8603 let peer_spec = TrustedPeerDescriptor::unsigned_with_pubkey(
8604 "mob/__mob_supervisor__",
8605 peer_pubkey.to_peer_id().as_str(),
8606 *peer_pubkey.as_bytes(),
8607 "inproc://mob/__mob_supervisor__",
8608 )
8609 .expect("valid peer spec");
8610 let runtime = Arc::new(
8611 meerkat_comms::CommsRuntime::inproc_only("bridge-response-typed-sender")
8612 .expect("runtime"),
8613 );
8614 let runtime_dyn: Arc<dyn CommsRuntime> = runtime.clone();
8615 add_test_projection_trust(runtime.as_ref(), peer_spec.clone(), "trust peer").await;
8616
8617 let payload_pubkey = meerkat_comms::Keypair::generate().public_key();
8618 let spoofed_payload_peer_id = PeerId::new();
8619 let command = BridgeCommand::ObserveMember(BridgeSupervisorPayload {
8620 supervisor: BridgePeerSpec {
8621 name: peer_spec.name.to_string(),
8622 peer_id: spoofed_payload_peer_id.as_str(),
8623 address: "inproc://mob/__mob_supervisor__".to_string(),
8624 pubkey: *payload_pubkey.as_bytes(),
8625 },
8626 epoch: 1,
8627 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
8628 });
8629 let id = InteractionId(Uuid::new_v4());
8630 let ingress = PeerIngressFact::peer(
8631 id,
8632 PeerInputClass::ActionableRequest,
8633 meerkat_core::PeerIngressKind::Request,
8634 Some(meerkat_core::PeerIngressAuthDecision::Exempt(
8635 meerkat_core::PeerIngressAuthExemption::SupervisorBridge,
8636 )),
8637 PeerIngressIdentity::new(
8638 peer_pubkey.to_peer_id(),
8639 peer_spec.name.as_str(),
8640 meerkat_core::PeerIngressConvention::Request {
8641 request_id: id.to_string(),
8642 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
8643 },
8644 )
8645 .with_signing_pubkey(*peer_pubkey.as_bytes()),
8646 );
8647 let spoofed_candidate =
8648 bridge_candidate_with_ingress(peer_spec.name.as_str(), &command, ingress);
8649 let route = resolve_bridge_response_route(&runtime_dyn, &spoofed_candidate)
8650 .await
8651 .expect("typed ingress route should ignore spoofed payload peer_id");
8652 assert_eq!(route.peer_id, peer_spec.peer_id);
8653 assert_ne!(route.peer_id, spoofed_payload_peer_id);
8654 assert_eq!(
8655 route.display_name.as_ref().map(|name| name.as_str()),
8656 Some(peer_spec.name.as_str())
8657 );
8658 }
8659
8660 #[tokio::test]
8661 async fn bridge_response_seeds_inbound_request_before_peer_response_send() {
8662 let suffix = Uuid::new_v4().simple().to_string();
8663 let member_name = format!("bridge-response-member-{suffix}");
8664 let supervisor_name = format!("bridge-response-supervisor-{suffix}");
8665 let member_runtime =
8666 Arc::new(meerkat_comms::CommsRuntime::inproc_only(&member_name).expect("member"));
8667 let supervisor_runtime = Arc::new(
8668 meerkat_comms::CommsRuntime::inproc_only(&supervisor_name).expect("supervisor"),
8669 );
8670 let peer_handle = Arc::new(CountingPeerInteractionHandle::default());
8671 let member_dsl = install_running_test_peer_comms_handle(member_runtime.as_ref());
8672 member_runtime.install_peer_request_response_authority(
8673 meerkat_comms::PeerRequestResponseAuthority::new(
8674 peer_handle.clone(),
8675 Arc::new(crate::handles::RuntimeInteractionStreamHandle::ephemeral()),
8676 ),
8677 );
8678 let supervisor_dsl = install_running_test_peer_comms_handle(supervisor_runtime.as_ref());
8679
8680 let supervisor_pubkey = supervisor_runtime.public_key();
8681 let supervisor_spec = TrustedPeerDescriptor::unsigned_with_pubkey(
8682 &supervisor_name,
8683 supervisor_pubkey.to_peer_id().as_str(),
8684 *supervisor_pubkey.as_bytes(),
8685 format!("inproc://{supervisor_name}"),
8686 )
8687 .expect("valid supervisor spec");
8688 add_test_projection_trust_with_dsl(
8689 member_runtime.as_ref(),
8690 Arc::clone(&member_dsl),
8691 supervisor_spec.clone(),
8692 1,
8693 "member should trust supervisor",
8694 )
8695 .await;
8696
8697 let member_pubkey = member_runtime.public_key();
8698 let member_spec = TrustedPeerDescriptor::unsigned_with_pubkey(
8699 &member_name,
8700 member_pubkey.to_peer_id().as_str(),
8701 *member_pubkey.as_bytes(),
8702 format!("inproc://{member_name}"),
8703 )
8704 .expect("valid member spec");
8705 add_test_projection_trust_with_dsl(
8706 supervisor_runtime.as_ref(),
8707 Arc::clone(&supervisor_dsl),
8708 member_spec,
8709 1,
8710 "supervisor should trust member",
8711 )
8712 .await;
8713
8714 let adapter = Arc::new(MeerkatMachine::ephemeral());
8715 let session_id = SessionId::new();
8716 adapter
8717 .register_session(session_id.clone())
8718 .await
8719 .expect("register session");
8720 adapter
8721 .stage_supervisor_bind(
8722 &session_id,
8723 supervisor_spec.name.to_string(),
8724 supervisor_spec.peer_id.as_str(),
8725 supervisor_spec.address.to_string(),
8726 signing_public_key_for_descriptor(&supervisor_spec),
8727 1,
8728 )
8729 .await
8730 .expect("pre-bind supervisor");
8731 let command = BridgeCommand::ObserveMember(BridgeSupervisorPayload {
8732 supervisor: BridgePeerSpec::from(supervisor_spec.clone()),
8733 epoch: 1,
8734 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
8735 });
8736 let interaction_id = InteractionId(Uuid::new_v4());
8737 let ingress = PeerIngressFact::peer(
8738 interaction_id,
8739 PeerInputClass::ActionableRequest,
8740 meerkat_core::PeerIngressKind::Request,
8741 Some(meerkat_core::PeerIngressAuthDecision::Exempt(
8742 meerkat_core::PeerIngressAuthExemption::SupervisorBridge,
8743 )),
8744 PeerIngressIdentity::new(
8745 supervisor_spec.peer_id,
8746 supervisor_spec.name.as_str(),
8747 meerkat_core::PeerIngressConvention::Request {
8748 request_id: interaction_id.to_string(),
8749 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
8750 },
8751 )
8752 .with_signing_pubkey(supervisor_spec.pubkey),
8753 );
8754 let candidate =
8755 bridge_candidate_with_ingress(supervisor_spec.name.as_str(), &command, ingress);
8756
8757 assert!(
8758 try_handle_supervisor_bridge_command(
8759 &adapter,
8760 &session_id,
8761 &(member_runtime.clone() as Arc<dyn CommsRuntime>),
8762 &candidate,
8763 )
8764 .await,
8765 "bridge handler must own ObserveMember"
8766 );
8767
8768 assert_eq!(
8769 peer_handle.request_received_count(),
8770 1,
8771 "bridge handler must seed inbound peer request state before replying"
8772 );
8773 assert_eq!(
8774 peer_handle.response_replied_count(),
8775 1,
8776 "real CommsRuntime::send(PeerResponse) should pass the inbound-state guard"
8777 );
8778 }
8779
8780 #[cfg(not(target_arch = "wasm32"))]
8781 #[tokio::test]
8782 async fn bind_member_bootstrap_stores_supervisor_address_not_member_address() {
8783 let suffix = Uuid::new_v4().simple().to_string();
8784 let member_name = format!("tcp-bootstrap-member-{suffix}");
8785 let supervisor_name = format!("tcp-bootstrap-supervisor-{suffix}");
8786
8787 let mut member_runtime =
8788 meerkat_comms::CommsRuntime::inproc_only(&member_name).expect("member runtime");
8789 member_runtime
8790 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
8791 .expect("configure member TCP listener");
8792 member_runtime
8793 .start_listeners()
8794 .await
8795 .expect("start member TCP listener");
8796 let member_runtime = Arc::new(member_runtime);
8797 install_test_peer_request_response_authority(
8798 &member_runtime,
8799 Arc::new(CountingPeerInteractionHandle::default()),
8800 );
8801
8802 let mut supervisor_runtime =
8803 meerkat_comms::CommsRuntime::inproc_only(&supervisor_name).expect("supervisor runtime");
8804 supervisor_runtime
8805 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
8806 .expect("configure supervisor TCP listener");
8807 supervisor_runtime
8808 .start_listeners()
8809 .await
8810 .expect("start supervisor TCP listener");
8811 let supervisor_runtime = Arc::new(supervisor_runtime);
8812 install_test_peer_request_response_authority(
8813 &supervisor_runtime,
8814 Arc::new(CountingPeerInteractionHandle::default()),
8815 );
8816
8817 let member_spec = trusted_tcp_peer_from_runtime(&member_name, &member_runtime);
8818 let supervisor_dsl = install_running_test_peer_comms_handle(supervisor_runtime.as_ref());
8819 add_test_projection_trust_with_dsl(
8820 supervisor_runtime.as_ref(),
8821 Arc::clone(&supervisor_dsl),
8822 member_spec.clone(),
8823 1,
8824 "supervisor trusts member target",
8825 )
8826 .await;
8827 let supervisor_spec =
8828 trusted_tcp_peer_from_runtime("mob/__mob_supervisor__", &supervisor_runtime);
8829
8830 let adapter = Arc::new(MeerkatMachine::ephemeral());
8831 let session_id = SessionId::new();
8832 adapter
8833 .register_session(session_id.clone())
8834 .await
8835 .expect("register session");
8836 adapter
8837 .test_install_session_peer_comms_handle_on_runtime(&session_id, member_runtime.as_ref())
8838 .await
8839 .expect("install adapter session peer-comms handle on member runtime");
8840
8841 let command = BridgeCommand::BindMember(
8842 meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
8843 supervisor: BridgePeerSpec::from(supervisor_spec.clone()),
8844 epoch: 1,
8845 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
8846 expected_peer_id: member_spec.peer_id.as_str(),
8847 expected_address: member_spec.address.to_string(),
8848 bootstrap_token: member_runtime.bridge_bootstrap_token().to_owned().into(),
8849 },
8850 );
8851 let request_receipt = supervisor_runtime
8852 .send(CommsCommand::PeerRequest {
8853 content_taint: None,
8854 to: PeerRoute::with_display_name(member_spec.peer_id, member_spec.name.clone()),
8855 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
8856 params: serde_json::to_value(&command).expect("serialize bridge command"),
8857 blocks: None,
8858 handling_mode: HandlingMode::Queue,
8859 stream: meerkat_core::comms::InputStreamMode::None,
8860 objective_id: None,
8861 })
8862 .await
8863 .expect("supervisor sends bootstrap BindMember over TCP");
8864 let meerkat_core::SendReceipt::PeerRequestSent { interaction_id, .. } = request_receipt
8865 else {
8866 panic!("expected PeerRequestSent receipt, got {request_receipt:?}");
8867 };
8868
8869 let candidate = drain_one_candidate(&member_runtime, "member BindMember request").await;
8870 assert_eq!(
8871 candidate.interaction.id, interaction_id,
8872 "member must receive the correlated bridge request"
8873 );
8874 assert!(
8875 try_handle_supervisor_bridge_command(
8876 &adapter,
8877 &session_id,
8878 &(member_runtime.clone() as Arc<dyn CommsRuntime>),
8879 &candidate,
8880 )
8881 .await,
8882 "bridge handler must own BindMember"
8883 );
8884
8885 let binding = adapter.supervisor_binding(&session_id).await;
8886 let SupervisorBinding::Bound {
8887 address: bound_address,
8888 ..
8889 } = binding
8890 else {
8891 panic!("bootstrap bind should leave supervisor bound, got {binding:?}");
8892 };
8893 assert_eq!(
8894 bound_address,
8895 supervisor_spec.address.to_string(),
8896 "supervisor binding must store the supervisor route, not the member route"
8897 );
8898 assert_ne!(
8899 bound_address,
8900 member_spec.address.to_string(),
8901 "storing the member address under the supervisor pubkey makes later bridge replies loop back to the member"
8902 );
8903
8904 let response =
8905 drain_one_candidate(&supervisor_runtime, "supervisor BindMember response").await;
8906 let InteractionContent::Response { result, .. } = response.interaction.content else {
8907 panic!(
8908 "expected bridge response, got {:?}",
8909 response.interaction.content
8910 );
8911 };
8912 let BridgeReply::BindMember(bind_response) =
8913 serde_json::from_value(result).expect("typed bridge reply")
8914 else {
8915 panic!("expected BindMember reply");
8916 };
8917 assert_eq!(bind_response.peer_id, member_spec.peer_id.as_str());
8918 assert_eq!(
8919 bind_response.address,
8920 canonicalize_bridge_address(&member_spec.address.to_string()),
8921 "BindMember reply still reports the member's canonical advertised address"
8922 );
8923 }
8924
8925 #[cfg(not(target_arch = "wasm32"))]
8926 #[tokio::test]
8927 async fn bind_member_idempotent_ack_repairs_tcp_supervisor_trust_before_reply() {
8928 let suffix = Uuid::new_v4().simple().to_string();
8929 let member_name = format!("tcp-bind-member-{suffix}");
8930 let supervisor_name = format!("tcp-supervisor-{suffix}");
8931
8932 let mut member_runtime =
8933 meerkat_comms::CommsRuntime::inproc_only(&member_name).expect("member runtime");
8934 member_runtime
8935 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
8936 .expect("configure member TCP listener");
8937 member_runtime
8938 .start_listeners()
8939 .await
8940 .expect("start member TCP listener");
8941 let member_runtime = Arc::new(member_runtime);
8942 let member_peer_handle = Arc::new(CountingPeerInteractionHandle::default());
8943 install_test_peer_request_response_authority(&member_runtime, member_peer_handle.clone());
8944
8945 let mut supervisor_runtime =
8946 meerkat_comms::CommsRuntime::inproc_only(&supervisor_name).expect("supervisor runtime");
8947 supervisor_runtime
8948 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
8949 .expect("configure supervisor TCP listener");
8950 supervisor_runtime
8951 .start_listeners()
8952 .await
8953 .expect("start supervisor TCP listener");
8954 let supervisor_runtime = Arc::new(supervisor_runtime);
8955 install_test_peer_request_response_authority(
8956 &supervisor_runtime,
8957 Arc::new(CountingPeerInteractionHandle::default()),
8958 );
8959
8960 let member_spec = trusted_tcp_peer_from_runtime(&member_name, &member_runtime);
8961 let supervisor_dsl = install_running_test_peer_comms_handle(supervisor_runtime.as_ref());
8962 add_test_projection_trust_with_dsl(
8963 supervisor_runtime.as_ref(),
8964 Arc::clone(&supervisor_dsl),
8965 member_spec.clone(),
8966 1,
8967 "supervisor trusts member target",
8968 )
8969 .await;
8970 let supervisor_spec =
8971 trusted_tcp_peer_from_runtime("mob/__mob_supervisor__", &supervisor_runtime);
8972
8973 let adapter = Arc::new(MeerkatMachine::ephemeral());
8974 let session_id = SessionId::new();
8975 adapter
8976 .register_session(session_id.clone())
8977 .await
8978 .expect("register session");
8979 adapter
8980 .stage_supervisor_bind(
8981 &session_id,
8982 supervisor_spec.name.to_string(),
8983 supervisor_spec.peer_id.to_string(),
8984 supervisor_spec.address.to_string(),
8985 signing_public_key_for_descriptor(&supervisor_spec),
8986 1,
8987 )
8988 .await
8989 .expect("pre-bind supervisor in DSL state");
8990 adapter
8991 .test_install_session_peer_comms_handle_on_runtime(&session_id, member_runtime.as_ref())
8992 .await
8993 .expect("install adapter session peer-comms handle on member runtime");
8994 assert!(
8995 member_runtime
8996 .peers()
8997 .await
8998 .iter()
8999 .all(|peer| peer.peer_id != supervisor_spec.peer_id),
9000 "fixture must start with DSL supervisor binding but no sendable runtime trust entry"
9001 );
9002
9003 let command = BridgeCommand::BindMember(
9004 meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
9005 supervisor: BridgePeerSpec::from(supervisor_spec.clone()),
9006 epoch: 1,
9007 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
9008 expected_peer_id: member_spec.peer_id.as_str(),
9009 expected_address: member_spec.address.to_string(),
9010 bootstrap_token: member_runtime.bridge_bootstrap_token().to_owned().into(),
9011 },
9012 );
9013 let request_receipt = supervisor_runtime
9014 .send(CommsCommand::PeerRequest {
9015 content_taint: None,
9016 to: PeerRoute::with_display_name(member_spec.peer_id, member_spec.name.clone()),
9017 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
9018 params: serde_json::to_value(&command).expect("serialize bridge command"),
9019 blocks: None,
9020 handling_mode: HandlingMode::Queue,
9021 stream: meerkat_core::comms::InputStreamMode::None,
9022 objective_id: None,
9023 })
9024 .await
9025 .expect("supervisor sends idempotent BindMember over TCP");
9026 let meerkat_core::SendReceipt::PeerRequestSent { interaction_id, .. } = request_receipt
9027 else {
9028 panic!("expected PeerRequestSent receipt, got {request_receipt:?}");
9029 };
9030
9031 let candidate = drain_one_candidate(&member_runtime, "member BindMember request").await;
9032 assert_eq!(
9033 candidate.interaction.id, interaction_id,
9034 "member must receive the correlated bridge request"
9035 );
9036 assert!(
9037 try_handle_supervisor_bridge_command(
9038 &adapter,
9039 &session_id,
9040 &(member_runtime.clone() as Arc<dyn CommsRuntime>),
9041 &candidate,
9042 )
9043 .await,
9044 "bridge handler must own BindMember"
9045 );
9046
9047 let response =
9048 drain_one_candidate(&supervisor_runtime, "supervisor BindMember response").await;
9049 assert_eq!(
9050 member_peer_handle.response_replied_count(),
9051 1,
9052 "target must send the terminal bridge response after repairing trust"
9053 );
9054 assert!(
9055 member_runtime
9056 .router()
9057 .is_private(&meerkat_comms::PubKey::new(supervisor_spec.pubkey)),
9058 "idempotent BindMember must repair the PRIVATE supervisor admission edge"
9059 );
9060 assert!(
9061 member_runtime
9062 .peers()
9063 .await
9064 .iter()
9065 .all(|peer| peer.peer_id != supervisor_spec.peer_id),
9066 "repaired supervisor edge is a control-plane route; it must not surface in public peers()"
9067 );
9068 let InteractionContent::Response {
9069 in_reply_to,
9070 status,
9071 result,
9072 ..
9073 } = response.interaction.content
9074 else {
9075 panic!(
9076 "expected bridge response, got {:?}",
9077 response.interaction.content
9078 );
9079 };
9080 assert_eq!(in_reply_to, interaction_id);
9081 assert_eq!(status, meerkat_core::interaction::ResponseStatus::Completed);
9082 let reply: BridgeReply = serde_json::from_value(result).expect("typed bridge reply");
9083 let BridgeReply::BindMember(bind_response) = reply else {
9084 panic!("expected BindMember reply, got {reply:?}");
9085 };
9086 assert_eq!(bind_response.peer_id, member_spec.peer_id.as_str());
9087 assert_eq!(
9088 bind_response.address,
9089 canonicalize_bridge_address(&member_spec.address.to_string())
9090 );
9091 }
9092
9093 #[cfg(not(target_arch = "wasm32"))]
9094 #[tokio::test]
9095 async fn authorized_bridge_command_repairs_tcp_supervisor_response_route_before_reply() {
9096 let suffix = Uuid::new_v4().simple().to_string();
9097 let member_name = format!("tcp-observe-member-{suffix}");
9098 let supervisor_name = format!("tcp-observe-supervisor-{suffix}");
9099
9100 let mut member_runtime =
9101 meerkat_comms::CommsRuntime::inproc_only(&member_name).expect("member runtime");
9102 member_runtime
9103 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
9104 .expect("configure member TCP listener");
9105 member_runtime
9106 .start_listeners()
9107 .await
9108 .expect("start member TCP listener");
9109 let member_runtime = Arc::new(member_runtime);
9110 let member_peer_handle = Arc::new(CountingPeerInteractionHandle::default());
9111 install_test_peer_request_response_authority(&member_runtime, member_peer_handle.clone());
9112
9113 let mut supervisor_runtime =
9114 meerkat_comms::CommsRuntime::inproc_only(&supervisor_name).expect("supervisor runtime");
9115 supervisor_runtime
9116 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
9117 .expect("configure supervisor TCP listener");
9118 supervisor_runtime
9119 .start_listeners()
9120 .await
9121 .expect("start supervisor TCP listener");
9122 let supervisor_runtime = Arc::new(supervisor_runtime);
9123 install_test_peer_request_response_authority(
9124 &supervisor_runtime,
9125 Arc::new(CountingPeerInteractionHandle::default()),
9126 );
9127
9128 let member_spec = trusted_tcp_peer_from_runtime(&member_name, &member_runtime);
9129 let supervisor_dsl = install_running_test_peer_comms_handle(supervisor_runtime.as_ref());
9130 add_test_projection_trust_with_dsl(
9131 supervisor_runtime.as_ref(),
9132 Arc::clone(&supervisor_dsl),
9133 member_spec.clone(),
9134 1,
9135 "supervisor trusts member target",
9136 )
9137 .await;
9138 let supervisor_spec =
9139 trusted_tcp_peer_from_runtime("mob/__mob_supervisor__", &supervisor_runtime);
9140
9141 let adapter = Arc::new(MeerkatMachine::ephemeral());
9142 let session_id = SessionId::new();
9143 adapter
9144 .register_session(session_id.clone())
9145 .await
9146 .expect("register session");
9147 adapter
9148 .stage_supervisor_bind(
9149 &session_id,
9150 supervisor_spec.name.to_string(),
9151 supervisor_spec.peer_id.as_str(),
9152 supervisor_spec.address.to_string(),
9153 signing_public_key_for_descriptor(&supervisor_spec),
9154 1,
9155 )
9156 .await
9157 .expect("pre-bind supervisor in DSL state");
9158 adapter
9159 .test_install_session_peer_comms_handle_on_runtime(&session_id, member_runtime.as_ref())
9160 .await
9161 .expect("install adapter session peer-comms handle on member runtime");
9162 assert!(
9163 member_runtime
9164 .peers()
9165 .await
9166 .iter()
9167 .all(|peer| peer.peer_id != supervisor_spec.peer_id),
9168 "fixture must start with authorized supervisor state but no public runtime route"
9169 );
9170
9171 let command = BridgeCommand::ObserveMember(BridgeSupervisorPayload {
9172 supervisor: BridgePeerSpec::from(supervisor_spec.clone()),
9173 epoch: 1,
9174 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
9175 });
9176 let interaction_id = InteractionId(Uuid::new_v4());
9177 let ingress = PeerIngressFact::peer(
9178 interaction_id,
9179 PeerInputClass::ActionableRequest,
9180 meerkat_core::PeerIngressKind::Request,
9181 Some(meerkat_core::PeerIngressAuthDecision::Required),
9182 PeerIngressIdentity::new(
9183 supervisor_spec.peer_id,
9184 supervisor_spec.name.as_str(),
9185 meerkat_core::PeerIngressConvention::Request {
9186 request_id: interaction_id.to_string(),
9187 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
9188 },
9189 )
9190 .with_signing_pubkey(supervisor_spec.pubkey),
9191 );
9192 let candidate =
9193 bridge_candidate_with_ingress(supervisor_spec.name.as_str(), &command, ingress);
9194
9195 assert!(
9196 try_handle_supervisor_bridge_command(
9197 &adapter,
9198 &session_id,
9199 &(member_runtime.clone() as Arc<dyn CommsRuntime>),
9200 &candidate,
9201 )
9202 .await,
9203 "bridge handler must own ObserveMember"
9204 );
9205 assert_eq!(
9206 member_peer_handle.response_replied_count(),
9207 1,
9208 "target must send the terminal bridge response after repairing supervisor route"
9209 );
9210
9211 let supervisor_pubkey = meerkat_comms::PubKey::new(supervisor_spec.pubkey);
9212 assert!(
9213 member_runtime.router().is_private(&supervisor_pubkey),
9214 "repaired supervisor response route must remain private"
9215 );
9216 assert!(
9217 member_runtime
9218 .peers()
9219 .await
9220 .iter()
9221 .all(|peer| peer.peer_id != supervisor_spec.peer_id),
9222 "private supervisor response route must not leak through public peers()"
9223 );
9224
9225 let response =
9226 drain_one_candidate(&supervisor_runtime, "supervisor ObserveMember response").await;
9227 let InteractionContent::Response {
9228 in_reply_to,
9229 status,
9230 result,
9231 ..
9232 } = response.interaction.content
9233 else {
9234 panic!(
9235 "expected bridge response, got {:?}",
9236 response.interaction.content
9237 );
9238 };
9239 assert_eq!(in_reply_to, interaction_id);
9240 assert_eq!(status, meerkat_core::interaction::ResponseStatus::Completed);
9241 assert!(
9242 matches!(
9243 serde_json::from_value::<BridgeReply>(result).expect("typed bridge reply"),
9244 BridgeReply::Observation(_)
9245 ),
9246 "expected ObserveMember response"
9247 );
9248 }
9249
9250 #[cfg(not(target_arch = "wasm32"))]
9251 #[tokio::test]
9252 async fn supervisor_rotation_v4_is_one_way_then_observed_from_new_authority() {
9253 let suffix = Uuid::new_v4().simple().to_string();
9254 let member_name = format!("tcp-authorize-member-{suffix}");
9255 let old_supervisor_name = format!("tcp-authorize-old-supervisor-{suffix}");
9256 let new_supervisor_name = format!("tcp-authorize-new-supervisor-{suffix}");
9257
9258 let mut member_runtime =
9259 meerkat_comms::CommsRuntime::inproc_only(&member_name).expect("member runtime");
9260 member_runtime
9261 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
9262 .expect("configure member TCP listener");
9263 member_runtime
9264 .start_listeners()
9265 .await
9266 .expect("start member TCP listener");
9267 let member_runtime = Arc::new(member_runtime);
9268 install_test_peer_request_response_authority(
9269 &member_runtime,
9270 Arc::new(CountingPeerInteractionHandle::default()),
9271 );
9272
9273 let mut old_supervisor_runtime =
9274 meerkat_comms::CommsRuntime::inproc_only(&old_supervisor_name)
9275 .expect("old supervisor runtime");
9276 old_supervisor_runtime
9277 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
9278 .expect("configure old supervisor TCP listener");
9279 old_supervisor_runtime
9280 .start_listeners()
9281 .await
9282 .expect("start old supervisor TCP listener");
9283 let old_supervisor_runtime = Arc::new(old_supervisor_runtime);
9284 install_test_peer_request_response_authority(
9285 &old_supervisor_runtime,
9286 Arc::new(CountingPeerInteractionHandle::default()),
9287 );
9288
9289 let new_supervisor_keypair = meerkat_comms::Keypair::generate();
9290 let mut new_supervisor_runtime =
9291 meerkat_comms::CommsRuntime::inproc_only_with_keypair_and_silent_intents(
9292 &new_supervisor_name,
9293 None,
9294 new_supervisor_keypair.clone(),
9295 Arc::new(HashSet::new()),
9296 )
9297 .expect("new supervisor runtime");
9298 new_supervisor_runtime
9299 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
9300 .expect("configure new supervisor TCP listener");
9301 new_supervisor_runtime
9302 .start_listeners()
9303 .await
9304 .expect("start new supervisor TCP listener");
9305 let new_supervisor_runtime = Arc::new(new_supervisor_runtime);
9306 install_test_peer_request_response_authority(
9307 &new_supervisor_runtime,
9308 Arc::new(CountingPeerInteractionHandle::default()),
9309 );
9310
9311 let member_spec = trusted_tcp_peer_from_runtime(&member_name, &member_runtime);
9312 let old_supervisor_dsl =
9313 install_running_test_peer_comms_handle(old_supervisor_runtime.as_ref());
9314 add_test_projection_trust_with_dsl(
9315 old_supervisor_runtime.as_ref(),
9316 Arc::clone(&old_supervisor_dsl),
9317 member_spec.clone(),
9318 1,
9319 "old supervisor trusts member target",
9320 )
9321 .await;
9322 let new_supervisor_dsl =
9323 install_running_test_peer_comms_handle(new_supervisor_runtime.as_ref());
9324 add_test_projection_trust_with_dsl(
9325 new_supervisor_runtime.as_ref(),
9326 Arc::clone(&new_supervisor_dsl),
9327 member_spec,
9328 1,
9329 "new supervisor trusts member target",
9330 )
9331 .await;
9332 let old_supervisor_spec =
9333 trusted_tcp_peer_from_runtime("mob/__mob_supervisor__", &old_supervisor_runtime);
9334 let new_supervisor_spec =
9335 trusted_tcp_peer_from_runtime("mob/__mob_supervisor__", &new_supervisor_runtime);
9336 let adapter = Arc::new(MeerkatMachine::ephemeral());
9342 let session_id = SessionId::new();
9343 adapter
9344 .register_session(session_id.clone())
9345 .await
9346 .expect("register session");
9347 adapter
9348 .stage_supervisor_bind(
9349 &session_id,
9350 old_supervisor_spec.name.to_string(),
9351 old_supervisor_spec.peer_id.as_str(),
9352 old_supervisor_spec.address.to_string(),
9353 signing_public_key_for_descriptor(&old_supervisor_spec),
9354 1,
9355 )
9356 .await
9357 .expect("pre-bind old supervisor in DSL state");
9358 adapter
9359 .test_install_session_peer_comms_handle_on_runtime(&session_id, member_runtime.as_ref())
9360 .await
9361 .expect("install adapter session peer-comms handle on member runtime");
9362 adapter
9363 .test_authorize_direct_comms_drain_runtime(
9364 &session_id,
9365 member_runtime.clone() as Arc<dyn CommsRuntime>,
9366 )
9367 .await
9368 .expect("install generated current member comms runtime fence");
9369 add_member_private_supervisor_trust_via_adapter(
9370 &adapter,
9371 &session_id,
9372 &member_runtime,
9373 &old_supervisor_spec,
9374 1,
9375 "member starts with old supervisor private route",
9376 )
9377 .await;
9378 assert!(
9379 member_runtime
9380 .router()
9381 .is_private(&meerkat_comms::PubKey::new(old_supervisor_spec.pubkey)),
9382 "member must start with a PRIVATE old supervisor route"
9383 );
9384
9385 let operation_id = SupervisorRotationOperationId::new();
9386 let delivery = serde_json::to_value(
9387 meerkat_contracts::wire::supervisor_bridge::BridgeSupervisorDelivery::SubmitSupervisorRotation(
9388 meerkat_contracts::wire::supervisor_bridge::BridgeSupervisorRotationSubmit {
9389 operation_id,
9390 target: BridgePeerSpec::from(new_supervisor_spec.clone()),
9391 target_epoch: 2,
9392 protocol_version: BridgeProtocolVersion::V4,
9393 },
9394 ),
9395 )
9396 .expect("serialize supervisor rotation delivery");
9397 let interaction_id = InteractionId(Uuid::new_v4());
9398 let ingress = PeerIngressFact::peer(
9399 interaction_id,
9400 PeerInputClass::ActionableMessage,
9401 meerkat_core::PeerIngressKind::Message,
9402 Some(meerkat_core::PeerIngressAuthDecision::Required),
9403 PeerIngressIdentity::new(
9404 old_supervisor_spec.peer_id,
9405 old_supervisor_spec.name.as_str(),
9406 meerkat_core::PeerIngressConvention::Message,
9407 )
9408 .with_signing_pubkey(old_supervisor_spec.pubkey),
9409 );
9410 let candidate = bridge_delivery_candidate_with_ingress(
9411 old_supervisor_spec.name.as_str(),
9412 &delivery,
9413 ingress,
9414 );
9415
9416 assert!(
9417 try_handle_supervisor_bridge_delivery(
9418 &adapter,
9419 &session_id,
9420 &(member_runtime.clone() as Arc<dyn CommsRuntime>),
9421 &candidate,
9422 )
9423 .await,
9424 "delivery handler must own SubmitSupervisorRotation"
9425 );
9426 tokio::time::timeout(Duration::from_secs(3), async {
9427 loop {
9428 let receipt = adapter
9429 .active_supervisor_rotation(&session_id)
9430 .await
9431 .expect("read rotation receipt")
9432 .expect("rotation receipt exists");
9433 if receipt.phase == crate::meerkat_machine::dsl::SupervisorRotationPhase::Completed
9434 {
9435 break;
9436 }
9437 tokio::time::sleep(Duration::from_millis(10)).await;
9438 }
9439 })
9440 .await
9441 .expect("rotation worker reaches terminal completion");
9442
9443 let observe = BridgeCommand::ObserveSupervisorRotation(
9444 meerkat_contracts::wire::supervisor_bridge::BridgeSupervisorRotationObserve {
9445 operation_id,
9446 observer: BridgePeerSpec::from(new_supervisor_spec.clone()),
9447 observer_epoch: 2,
9448 protocol_version: BridgeProtocolVersion::V4,
9449 },
9450 );
9451 let probe_name = format!("{new_supervisor_name}/pending-supervisor-probe");
9457 let mut probe_runtime =
9458 meerkat_comms::CommsRuntime::inproc_only_with_keypair_and_silent_intents(
9459 &probe_name,
9460 None,
9461 new_supervisor_keypair,
9462 Arc::new(HashSet::new()),
9463 )
9464 .expect("alternate-authority probe runtime");
9465 probe_runtime
9466 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
9467 .expect("configure alternate-authority probe TCP listener");
9468 probe_runtime
9469 .start_listeners()
9470 .await
9471 .expect("start alternate-authority probe TCP listener");
9472 let probe_runtime = Arc::new(probe_runtime);
9473 install_test_peer_request_response_authority(
9474 &probe_runtime,
9475 Arc::new(CountingPeerInteractionHandle::default()),
9476 );
9477 let probe_dsl = install_running_test_peer_comms_handle(probe_runtime.as_ref());
9478 let member_spec = trusted_tcp_peer_from_runtime(&member_name, &member_runtime);
9479 add_test_projection_trust_with_dsl(
9480 probe_runtime.as_ref(),
9481 Arc::clone(&probe_dsl),
9482 member_spec.clone(),
9483 1,
9484 "alternate-authority probe trusts member target",
9485 )
9486 .await;
9487 let reply_endpoint = PeerAddress::parse(probe_runtime.advertised_address())
9488 .expect("probe advertises a typed TCP reply endpoint");
9489 assert_ne!(
9490 reply_endpoint.to_string(),
9491 new_supervisor_spec.address.to_string(),
9492 "regression requires the durable authority and probe listeners to differ"
9493 );
9494 let receipt = probe_runtime
9495 .send_peer_request_at_endpoint(
9496 PeerRoute::with_display_name(member_spec.peer_id, member_spec.name.clone()),
9497 SUPERVISOR_BRIDGE_INTENT,
9498 serde_json::to_value(&observe).expect("serialize observe command"),
9499 reply_endpoint.clone(),
9500 )
9501 .await
9502 .expect("send signed observation from alternate-authority probe");
9503 let SendReceipt::PeerRequestSent { envelope_id, .. } = receipt else {
9504 panic!("expected peer request receipt, got {receipt:?}");
9505 };
9506 let observe_id = InteractionId(envelope_id);
9507 let observe_candidate =
9508 drain_one_candidate(&member_runtime, "alternate-authority observation request").await;
9509 assert_eq!(observe_candidate.interaction.id, observe_id);
9510 assert_eq!(
9511 observe_candidate.ingress.declared_reply_endpoint,
9512 Some(reply_endpoint),
9513 "the authenticated request reply endpoint must survive ingress classification"
9514 );
9515 assert!(
9516 try_handle_supervisor_bridge_command(
9517 &adapter,
9518 &session_id,
9519 &(member_runtime.clone() as Arc<dyn CommsRuntime>),
9520 &observe_candidate,
9521 )
9522 .await,
9523 "bridge handler must own ObserveSupervisorRotation"
9524 );
9525
9526 let response = drain_one_candidate(
9527 &probe_runtime,
9528 "alternate-authority supervisor rotation observation",
9529 )
9530 .await;
9531 let InteractionContent::Response {
9532 in_reply_to,
9533 status,
9534 result,
9535 ..
9536 } = response.interaction.content
9537 else {
9538 panic!(
9539 "expected authorize bridge response, got {:?}",
9540 response.interaction.content
9541 );
9542 };
9543 assert_eq!(in_reply_to, observe_id);
9544 assert_eq!(status, meerkat_core::interaction::ResponseStatus::Completed);
9545 assert!(matches!(
9546 serde_json::from_value::<BridgeReply>(result).expect("typed bridge reply"),
9547 BridgeReply::SupervisorRotation(
9548 BridgeSupervisorRotationObservation::Found {
9549 state: BridgeSupervisorRotationState::Completed { receipt }
9550 }
9551 ) if receipt.operation_id == operation_id
9552 && receipt.target.target.peer_id == new_supervisor_spec.peer_id.as_str()
9553 ));
9554 let peers = member_runtime.peers().await;
9555 assert!(
9558 peers
9559 .iter()
9560 .all(|peer| peer.peer_id != old_supervisor_spec.peer_id),
9561 "old supervisor route must be removed after rotation"
9562 );
9563 assert!(
9564 peers
9565 .iter()
9566 .all(|peer| peer.peer_id != new_supervisor_spec.peer_id),
9567 "new supervisor route must stay private (not in public peers())"
9568 );
9569 assert!(
9570 member_runtime
9571 .router()
9572 .is_private(&meerkat_comms::PubKey::new(new_supervisor_spec.pubkey)),
9573 "new supervisor private route must remain installed after rotation"
9574 );
9575 assert!(
9576 !member_runtime
9577 .router()
9578 .is_private(&meerkat_comms::PubKey::new(old_supervisor_spec.pubkey)),
9579 "old supervisor private route must be fully removed after rotation"
9580 );
9581 }
9582
9583 #[cfg(not(target_arch = "wasm32"))]
9584 #[tokio::test]
9585 async fn revoke_supervisor_replies_after_removing_supervisor_route() {
9586 let suffix = Uuid::new_v4().simple().to_string();
9587 let member_name = format!("tcp-revoke-member-{suffix}");
9588 let supervisor_name = format!("tcp-revoke-supervisor-{suffix}");
9589
9590 let mut member_runtime =
9591 meerkat_comms::CommsRuntime::inproc_only(&member_name).expect("member runtime");
9592 member_runtime
9593 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
9594 .expect("configure member TCP listener");
9595 member_runtime
9596 .start_listeners()
9597 .await
9598 .expect("start member TCP listener");
9599 let member_runtime = Arc::new(member_runtime);
9600 install_test_peer_request_response_authority(
9601 &member_runtime,
9602 Arc::new(CountingPeerInteractionHandle::default()),
9603 );
9604
9605 let mut supervisor_runtime =
9606 meerkat_comms::CommsRuntime::inproc_only(&supervisor_name).expect("supervisor runtime");
9607 supervisor_runtime
9608 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from(([127, 0, 0, 1], 0)))
9609 .expect("configure supervisor TCP listener");
9610 supervisor_runtime
9611 .start_listeners()
9612 .await
9613 .expect("start supervisor TCP listener");
9614 let supervisor_runtime = Arc::new(supervisor_runtime);
9615 install_test_peer_request_response_authority(
9616 &supervisor_runtime,
9617 Arc::new(CountingPeerInteractionHandle::default()),
9618 );
9619
9620 let member_spec = trusted_tcp_peer_from_runtime(&member_name, &member_runtime);
9621 let supervisor_dsl = install_running_test_peer_comms_handle(supervisor_runtime.as_ref());
9622 add_test_projection_trust_with_dsl(
9623 supervisor_runtime.as_ref(),
9624 Arc::clone(&supervisor_dsl),
9625 member_spec,
9626 1,
9627 "supervisor trusts member target",
9628 )
9629 .await;
9630 let supervisor_spec =
9631 trusted_tcp_peer_from_runtime("mob/__mob_supervisor__", &supervisor_runtime);
9632 let adapter = Arc::new(MeerkatMachine::ephemeral());
9637 let session_id = SessionId::new();
9638 adapter
9639 .register_session(session_id.clone())
9640 .await
9641 .expect("register session");
9642 adapter
9643 .stage_supervisor_bind(
9644 &session_id,
9645 supervisor_spec.name.to_string(),
9646 supervisor_spec.peer_id.as_str(),
9647 supervisor_spec.address.to_string(),
9648 signing_public_key_for_descriptor(&supervisor_spec),
9649 1,
9650 )
9651 .await
9652 .expect("pre-bind supervisor in DSL state");
9653 adapter
9654 .test_install_session_peer_comms_handle_on_runtime(&session_id, member_runtime.as_ref())
9655 .await
9656 .expect("install adapter session peer-comms handle on member runtime");
9657 add_member_private_supervisor_trust_via_adapter(
9658 &adapter,
9659 &session_id,
9660 &member_runtime,
9661 &supervisor_spec,
9662 1,
9663 "member starts with supervisor private route",
9664 )
9665 .await;
9666 assert!(
9667 member_runtime
9668 .router()
9669 .is_private(&meerkat_comms::PubKey::new(supervisor_spec.pubkey)),
9670 "member must start with a PRIVATE supervisor route"
9671 );
9672
9673 let command = BridgeCommand::RevokeSupervisor(BridgeSupervisorPayload {
9674 supervisor: BridgePeerSpec::from(supervisor_spec.clone()),
9675 epoch: 1,
9676 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
9677 });
9678 let interaction_id = InteractionId(Uuid::new_v4());
9679 let ingress = PeerIngressFact::peer(
9680 interaction_id,
9681 PeerInputClass::ActionableRequest,
9682 meerkat_core::PeerIngressKind::Request,
9683 Some(meerkat_core::PeerIngressAuthDecision::Required),
9684 PeerIngressIdentity::new(
9685 supervisor_spec.peer_id,
9686 supervisor_spec.name.as_str(),
9687 meerkat_core::PeerIngressConvention::Request {
9688 request_id: interaction_id.to_string(),
9689 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
9690 },
9691 )
9692 .with_signing_pubkey(supervisor_spec.pubkey),
9693 );
9694 let candidate =
9695 bridge_candidate_with_ingress(supervisor_spec.name.as_str(), &command, ingress);
9696
9697 assert!(
9698 try_handle_supervisor_bridge_command(
9699 &adapter,
9700 &session_id,
9701 &(member_runtime.clone() as Arc<dyn CommsRuntime>),
9702 &candidate,
9703 )
9704 .await,
9705 "bridge handler must own RevokeSupervisor"
9706 );
9707
9708 let response =
9709 drain_one_candidate(&supervisor_runtime, "supervisor RevokeSupervisor response").await;
9710 let InteractionContent::Response {
9711 in_reply_to,
9712 status,
9713 result,
9714 ..
9715 } = response.interaction.content
9716 else {
9717 panic!(
9718 "expected revoke bridge response, got {:?}",
9719 response.interaction.content
9720 );
9721 };
9722 assert_eq!(in_reply_to, interaction_id);
9723 assert_eq!(status, meerkat_core::interaction::ResponseStatus::Completed);
9724 assert!(matches!(
9725 serde_json::from_value::<BridgeReply>(result).expect("typed bridge reply"),
9726 BridgeReply::Ack(BridgeAck { ok: true })
9727 ));
9728 assert!(
9729 member_runtime
9730 .peers()
9731 .await
9732 .iter()
9733 .all(|peer| peer.peer_id != supervisor_spec.peer_id),
9734 "supervisor route must be removed after revoke"
9735 );
9736 assert!(
9737 !member_runtime
9738 .router()
9739 .is_private(&meerkat_comms::PubKey::new(supervisor_spec.pubkey)),
9740 "supervisor private admission edge must be fully removed after revoke"
9741 );
9742 }
9743
9744 #[cfg(not(target_arch = "wasm32"))]
9745 async fn drain_one_candidate(
9746 runtime: &Arc<meerkat_comms::CommsRuntime>,
9747 label: &str,
9748 ) -> PeerInputCandidate {
9749 for _ in 0..40 {
9750 let candidates = runtime.drain_peer_input_candidates().await;
9751 if let Some(candidate) = candidates.into_iter().next() {
9752 return candidate;
9753 }
9754 let notify = runtime.inbox_notify();
9755 let notified = notify.notified();
9756 let _ = tokio::time::timeout(std::time::Duration::from_millis(25), notified).await;
9757 }
9758 panic!("timed out waiting for {label}");
9759 }
9760
9761 #[test]
9762 fn bridge_response_route_source_does_not_use_interaction_from_ladder() {
9763 let source = include_str!("comms_drain.rs");
9764 let route_start = source
9765 .find("async fn resolve_bridge_response_route")
9766 .expect("route function exists");
9767 let route_end = source[route_start..]
9768 .find("async fn resolve_peer_route")
9769 .map(|offset| route_start + offset)
9770 .expect("peer route function follows bridge response route");
9771 let route_source = &source[route_start..route_end];
9772 let forbidden_from = ["candidate", "interaction", "from"].join(".");
9773 assert!(
9774 !route_source.contains(&forbidden_from),
9775 "bridge response routing must use typed ingress facts, not candidate.interaction.from"
9776 );
9777
9778 for removed_helper in [
9779 ["bridge_response", "route_from_sender"].join("_"),
9780 ["resolve_bridge", "display_name_sender_route"].join("_"),
9781 ] {
9782 assert!(
9783 !source.contains(&format!("fn {removed_helper}")),
9784 "bridge response routing must not revive {removed_helper}"
9785 );
9786 }
9787 }
9788
9789 #[test]
9790 fn comms_drain_registers_inbox_waiter_before_draining() {
9791 let source = include_str!("comms_drain.rs");
9799 let enable = source
9800 .find("notified.as_mut().enable();")
9801 .expect("drain must enable() its pinned inbox waiter");
9802 let drain = source
9803 .find("comms_runtime.drain_classified_inbox_interactions().await")
9804 .expect("drain must consume the typed classified inbox path");
9805 assert!(
9806 enable < drain,
9807 "the inbox waiter must be registered (enable) BEFORE the drain, not after"
9808 );
9809 }
9810
9811 #[tokio::test]
9812 async fn notify_waiters_is_lost_without_prior_registration() {
9813 use std::sync::Arc;
9819 use tokio::sync::Notify;
9820
9821 let notify = Arc::new(Notify::new());
9822 notify.notify_waiters(); let late = notify.notified();
9824 assert!(
9825 tokio::time::timeout(std::time::Duration::from_millis(50), late)
9826 .await
9827 .is_err(),
9828 "a notify_waiters() with no prior registration must be lost"
9829 );
9830
9831 let mut early = std::pin::pin!(notify.notified());
9834 early.as_mut().enable();
9835 notify.notify_waiters();
9836 tokio::time::timeout(std::time::Duration::from_millis(100), early)
9837 .await
9838 .expect("a waiter enabled before notify_waiters() must observe it");
9839 }
9840
9841 struct BootstrapRuntime {
9842 peer_id: String,
9843 address: String,
9844 bootstrap_token: Option<String>,
9845 inbox_notify: Arc<tokio::sync::Notify>,
9846 add_trusted_peer_errors: HashMap<String, String>,
9847 remove_trusted_peer_errors: HashMap<String, String>,
9848 trusted_peer_ids: Arc<tokio::sync::Mutex<HashSet<String>>>,
9849 trusted_peers: Arc<tokio::sync::Mutex<HashMap<String, TrustedPeerDescriptor>>>,
9850 sent_commands: Arc<tokio::sync::Mutex<Vec<CommsCommand>>>,
9851 peer_handle: Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>>,
9852 peer_request_response_handle: Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>>,
9853 completed_count: Arc<std::sync::atomic::AtomicUsize>,
9854 add_mutates_then_hangs: Arc<std::sync::atomic::AtomicBool>,
9855 add_mutates_then_hangs_once_for: Arc<tokio::sync::Mutex<HashSet<String>>>,
9856 remove_hangs_remaining: Arc<std::sync::atomic::AtomicUsize>,
9857 remove_mutates_then_hangs_once_for: Arc<tokio::sync::Mutex<HashSet<String>>>,
9858 remove_failures_remaining: Arc<std::sync::atomic::AtomicUsize>,
9859 remove_attempts: Arc<std::sync::atomic::AtomicUsize>,
9860 staged_reply_endpoints: Arc<std::sync::Mutex<Vec<(PeerId, [u8; 32], String)>>>,
9861 staged_correlated_reply_endpoints:
9862 Arc<std::sync::Mutex<Vec<(PeerId, InteractionId, [u8; 32], PeerAddress)>>>,
9863 unstaged_correlated_reply_endpoints: Arc<std::sync::Mutex<Vec<(PeerId, InteractionId)>>>,
9864 }
9865
9866 #[async_trait::async_trait]
9867 impl CommsRuntime for BootstrapRuntime {
9868 fn peer_id(&self) -> Option<PeerId> {
9869 PeerId::parse(&self.peer_id).ok()
9870 }
9871
9872 fn public_key(&self) -> Option<String> {
9873 test_public_key_for_peer_id(&self.peer_id)
9874 }
9875
9876 fn public_key_bytes(&self) -> Option<[u8; 32]> {
9877 test_pubkey_bytes_for_peer_id(&self.peer_id)
9878 }
9879
9880 fn comms_name(&self) -> Option<String> {
9881 Some(test_comms_name_from_address(&self.address))
9882 }
9883
9884 fn advertised_address(&self) -> Option<String> {
9885 Some(self.address.clone())
9886 }
9887
9888 fn bridge_bootstrap_token(&self) -> Option<String> {
9889 self.bootstrap_token.clone()
9890 }
9891
9892 async fn drain_messages(&self) -> Vec<String> {
9893 Vec::new()
9894 }
9895
9896 fn inbox_notify(&self) -> Arc<tokio::sync::Notify> {
9897 self.inbox_notify.clone()
9898 }
9899
9900 fn peer_interaction_handle(
9901 &self,
9902 ) -> Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>> {
9903 self.peer_handle.clone()
9904 }
9905
9906 fn peer_request_response_authority_handle(
9907 &self,
9908 ) -> Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>> {
9909 self.peer_request_response_handle.clone()
9910 }
9911
9912 fn mark_interaction_complete(&self, _id: &InteractionId) {
9913 self.completed_count
9914 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
9915 }
9916
9917 async fn stage_declared_reply_endpoint(
9918 &self,
9919 dest: PeerId,
9920 signer_pubkey: [u8; 32],
9921 declared_address: String,
9922 ) -> Result<(), SendError> {
9923 self.staged_reply_endpoints
9924 .lock()
9925 .expect("staged reply endpoints mutex")
9926 .push((dest, signer_pubkey, declared_address));
9927 Ok(())
9928 }
9929
9930 async fn stage_correlated_reply_endpoint(
9931 &self,
9932 dest: PeerId,
9933 in_reply_to: InteractionId,
9934 signer_pubkey: [u8; 32],
9935 declared_endpoint: PeerAddress,
9936 ) -> Result<(), SendError> {
9937 self.staged_correlated_reply_endpoints
9938 .lock()
9939 .expect("staged correlated reply endpoints mutex")
9940 .push((dest, in_reply_to, signer_pubkey, declared_endpoint));
9941 Ok(())
9942 }
9943
9944 async fn unstage_correlated_reply_endpoint(
9945 &self,
9946 dest: PeerId,
9947 in_reply_to: InteractionId,
9948 ) -> Result<(), SendError> {
9949 self.unstaged_correlated_reply_endpoints
9950 .lock()
9951 .expect("unstaged correlated reply endpoints mutex")
9952 .push((dest, in_reply_to));
9953 Ok(())
9954 }
9955
9956 async fn apply_trust_mutation(
9957 &self,
9958 mutation: CommsTrustMutation,
9959 ) -> Result<CommsTrustMutationResult, SendError> {
9960 match mutation {
9961 CommsTrustMutation::AddPrivateTrustedPeer { peer, authority } => {
9962 authority
9963 .validate_private_add(self.peer_id(), &peer)
9964 .map_err(SendError::Validation)?;
9965 let peer_id_str = peer.peer_id.as_str();
9966 if let Some(message) = self.add_trusted_peer_errors.get(&peer_id_str) {
9967 return Err(SendError::Internal(message.clone()));
9968 }
9969 let created = self.trusted_peer_ids.lock().await.insert(peer_id_str);
9970 let mut mutate_then_hang = self.add_mutates_then_hangs_once_for.lock().await;
9971 let targeted_mutate_then_hang = mutate_then_hang.remove(&peer.peer_id.as_str());
9972 drop(mutate_then_hang);
9973 if self
9974 .add_mutates_then_hangs
9975 .load(std::sync::atomic::Ordering::SeqCst)
9976 || targeted_mutate_then_hang
9977 {
9978 std::future::pending::<()>().await;
9979 }
9980 Ok(CommsTrustMutationResult::Added { created })
9981 }
9982 CommsTrustMutation::RemovePrivateTrustedPeer { peer_id, authority } => {
9983 let parsed_peer_id = PeerId::parse(&peer_id)
9984 .map_err(|error| SendError::Validation(error.to_string()))?;
9985 authority
9986 .validate_private_remove(self.peer_id(), parsed_peer_id)
9987 .map_err(SendError::Validation)?;
9988 self.remove_attempts
9989 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
9990 let mut mutate_then_hang = self.remove_mutates_then_hangs_once_for.lock().await;
9991 let targeted_mutate_then_hang = mutate_then_hang.remove(&peer_id);
9992 drop(mutate_then_hang);
9993 if targeted_mutate_then_hang {
9994 self.trusted_peer_ids.lock().await.remove(&peer_id);
9995 std::future::pending::<()>().await;
9996 }
9997 if self
9998 .remove_hangs_remaining
9999 .fetch_update(
10000 std::sync::atomic::Ordering::SeqCst,
10001 std::sync::atomic::Ordering::SeqCst,
10002 |remaining| remaining.checked_sub(1),
10003 )
10004 .is_ok()
10005 {
10006 std::future::pending::<()>().await;
10007 }
10008 if self
10009 .remove_failures_remaining
10010 .fetch_update(
10011 std::sync::atomic::Ordering::SeqCst,
10012 std::sync::atomic::Ordering::SeqCst,
10013 |remaining| remaining.checked_sub(1),
10014 )
10015 .is_ok()
10016 {
10017 return Err(SendError::Internal(
10018 "synthetic one-shot remove failure".to_string(),
10019 ));
10020 }
10021 match self.remove_trusted_peer_errors.get(&peer_id) {
10022 Some(message) => Err(SendError::Internal(message.clone())),
10023 None => Ok(CommsTrustMutationResult::Removed {
10024 removed: self.trusted_peer_ids.lock().await.remove(&peer_id),
10025 }),
10026 }
10027 }
10028 _ => Err(SendError::Unsupported(
10029 "test runtime only supports generated private trust".into(),
10030 )),
10031 }
10032 }
10033
10034 async fn send(&self, cmd: CommsCommand) -> Result<meerkat_core::SendReceipt, SendError> {
10035 let receipt = match &cmd {
10036 CommsCommand::PeerResponse { in_reply_to, .. } => {
10037 meerkat_core::SendReceipt::PeerResponseSent {
10038 envelope_id: Uuid::new_v4(),
10039 in_reply_to: *in_reply_to,
10040 }
10041 }
10042 other => {
10043 return Err(SendError::Unsupported(format!(
10044 "BootstrapRuntime only records peer responses, got {}",
10045 other.command_kind()
10046 )));
10047 }
10048 };
10049 self.sent_commands.lock().await.push(cmd);
10050 Ok(receipt)
10051 }
10052
10053 async fn peer_ingress_runtime_snapshot(
10054 &self,
10055 ) -> Result<
10056 meerkat_core::interaction::PeerIngressRuntimeSnapshot,
10057 meerkat_core::CommsCapabilityError,
10058 > {
10059 Ok(meerkat_core::interaction::PeerIngressRuntimeSnapshot {
10060 self_peer_id: self
10061 .peer_id()
10062 .unwrap_or_else(|| PeerId::parse(PEER_ID_RECEIVER).expect("valid peer id")),
10063 auth_required: true,
10064 authority_phase: meerkat_core::interaction::PeerIngressAuthorityPhase::Received,
10065 trusted_peers: self.trusted_peers.lock().await.values().cloned().collect(),
10066 submission_queue_len: 0,
10067 queue: meerkat_core::interaction::PeerIngressQueueSnapshot::default(),
10068 })
10069 }
10070 }
10071
10072 fn bootstrap_runtime(
10073 peer_id: &str,
10074 address: &str,
10075 bootstrap_token: Option<&str>,
10076 ) -> BootstrapRuntime {
10077 let peer_handle = Arc::new(CountingPeerInteractionHandle::default());
10078 let peer_handle: Arc<dyn meerkat_core::handles::PeerInteractionHandle> = peer_handle;
10079 BootstrapRuntime {
10080 peer_id: peer_id.to_string(),
10081 address: address.to_string(),
10082 bootstrap_token: bootstrap_token.map(ToString::to_string),
10083 inbox_notify: Arc::new(tokio::sync::Notify::new()),
10084 add_trusted_peer_errors: HashMap::new(),
10085 remove_trusted_peer_errors: HashMap::new(),
10086 trusted_peer_ids: Arc::new(tokio::sync::Mutex::new(HashSet::new())),
10087 trusted_peers: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
10088 sent_commands: Arc::new(tokio::sync::Mutex::new(Vec::new())),
10089 peer_handle: Some(peer_handle.clone()),
10090 peer_request_response_handle: Some(peer_handle),
10091 completed_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
10092 add_mutates_then_hangs: Arc::new(std::sync::atomic::AtomicBool::new(false)),
10093 add_mutates_then_hangs_once_for: Arc::new(tokio::sync::Mutex::new(HashSet::new())),
10094 remove_hangs_remaining: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
10095 remove_mutates_then_hangs_once_for: Arc::new(tokio::sync::Mutex::new(HashSet::new())),
10096 remove_failures_remaining: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
10097 remove_attempts: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
10098 staged_reply_endpoints: Arc::new(std::sync::Mutex::new(Vec::new())),
10099 staged_correlated_reply_endpoints: Arc::new(std::sync::Mutex::new(Vec::new())),
10100 unstaged_correlated_reply_endpoints: Arc::new(std::sync::Mutex::new(Vec::new())),
10101 }
10102 }
10103
10104 #[tokio::test]
10105 async fn supervisor_rotation_task_slot_rejects_live_replacement_carrier() {
10106 struct DropSignal(Arc<std::sync::atomic::AtomicBool>);
10107
10108 impl Drop for DropSignal {
10109 fn drop(&mut self) {
10110 self.0.store(true, std::sync::atomic::Ordering::SeqCst);
10111 }
10112 }
10113
10114 let slot = crate::meerkat_machine::SupervisorRotationTaskSlot::new();
10115 let runtime_a: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
10116 PEER_ID_RECEIVER,
10117 "inproc://rotation-worker-a",
10118 Some("token"),
10119 ));
10120 let runtime_b: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
10121 PEER_ID_RECEIVER,
10122 "inproc://rotation-worker-b",
10123 Some("token"),
10124 ));
10125 let first_dropped = Arc::new(std::sync::atomic::AtomicBool::new(false));
10126 let first_started = Arc::new(tokio::sync::Notify::new());
10127 let first = tokio::spawn({
10128 let first_dropped = Arc::clone(&first_dropped);
10129 let first_started = Arc::clone(&first_started);
10130 async move {
10131 let _drop_signal = DropSignal(first_dropped);
10132 first_started.notify_one();
10133 std::future::pending::<()>().await;
10134 }
10135 });
10136 first_started.notified().await;
10137 assert!(
10138 slot.install("operation-a".to_string(), runtime_a, first)
10139 .await
10140 );
10141
10142 let replacement_dropped = Arc::new(std::sync::atomic::AtomicBool::new(false));
10143 let replacement = tokio::spawn({
10144 let drop_signal = DropSignal(Arc::clone(&replacement_dropped));
10145 async move {
10146 let _drop_signal = drop_signal;
10147 std::future::pending::<()>().await;
10148 }
10149 });
10150 assert!(
10151 !slot
10152 .install("operation-b".to_string(), runtime_b, replacement)
10153 .await,
10154 "a live carrier must never be displaced by last-writer-wins installation"
10155 );
10156 assert!(
10157 !first_dropped.load(std::sync::atomic::Ordering::SeqCst),
10158 "the incumbent carrier remains authoritative until its owning runtime is quiesced"
10159 );
10160 assert!(
10161 replacement_dropped.load(std::sync::atomic::Ordering::SeqCst),
10162 "the rejected candidate carrier must be aborted and joined"
10163 );
10164 slot.abort_and_wait().await;
10165 assert!(
10166 first_dropped.load(std::sync::atomic::Ordering::SeqCst),
10167 "explicit quiescence must abort and join the incumbent carrier"
10168 );
10169 }
10170
10171 fn lifecycle_candidate(
10172 class: PeerInputClass,
10173 intent: &str,
10174 params: serde_json::Value,
10175 ) -> PeerInputCandidate {
10176 let id = InteractionId(Uuid::new_v4());
10177 let lifecycle_kind = match class {
10178 PeerInputClass::PeerLifecycleRetired => {
10179 meerkat_core::comms::PeerLifecycleKind::PeerRetired
10180 }
10181 PeerInputClass::PeerLifecycleUnwired => {
10182 meerkat_core::comms::PeerLifecycleKind::PeerUnwired
10183 }
10184 _ => meerkat_core::comms::PeerLifecycleKind::PeerAdded,
10185 };
10186 PeerInputCandidate {
10187 interaction: InboxInteraction {
10188 objective_id: None,
10189 sender_taint: None,
10190 id,
10191 from_route: None,
10192 from: "test-mob/__mob_supervisor__".to_string(),
10193 content: InteractionContent::Request {
10194 intent: intent.to_string(),
10195 params,
10196 blocks: None,
10197 },
10198 rendered_text: String::new(),
10199 handling_mode: HandlingMode::Queue,
10200 render_metadata: None,
10201 },
10202 ingress: PeerIngressFact::peer(
10203 id,
10204 class,
10205 meerkat_core::PeerIngressKind::Request,
10206 Some(meerkat_core::PeerIngressAuthDecision::Required),
10207 PeerIngressIdentity::new(
10208 PeerId::new(),
10209 "test-mob/__mob_supervisor__",
10210 meerkat_core::PeerIngressConvention::Lifecycle {
10211 kind: lifecycle_kind,
10212 peer: "peer-1".to_string(),
10213 },
10214 ),
10215 ),
10216 lifecycle_peer: Some("peer-1".to_string()),
10217 response_terminality: None,
10218 }
10219 }
10220
10221 #[test]
10222 fn completed_outcome_maps_structured_output_to_interaction_complete() {
10223 let interaction_id = InteractionId(Uuid::new_v4());
10224 let event = interaction_terminal_event(
10225 interaction_id,
10226 CompletionOutcome::Completed(Box::new(meerkat_core::RunResult {
10227 text: "{\"answer\":42}".to_string(),
10228 session_id: meerkat_core::SessionId::new(),
10229 usage: meerkat_core::Usage::default(),
10230 turns: 1,
10231 tool_calls: 0,
10232 terminal_cause_kind: None,
10233 structured_output: Some(json!({"answer": 42})),
10234 extraction_error: None,
10235 schema_warnings: None,
10236 skill_diagnostics: None,
10237 })),
10238 );
10239
10240 match event {
10241 AgentEvent::InteractionComplete {
10242 interaction_id: actual_id,
10243 result,
10244 structured_output,
10245 } => {
10246 assert_eq!(actual_id, interaction_id);
10247 assert_eq!(result, "{\"answer\":42}");
10248 assert_eq!(structured_output, Some(json!({"answer": 42})));
10249 }
10250 other => panic!("expected InteractionComplete, got {other:?}"),
10251 }
10252 }
10253
10254 #[test]
10255 fn extraction_failure_maps_to_typed_interaction_failure() {
10256 let interaction_id = InteractionId(Uuid::new_v4());
10257 let event = interaction_terminal_event(
10258 interaction_id,
10259 CompletionOutcome::Completed(Box::new(meerkat_core::RunResult {
10260 text: "main output".to_string(),
10261 session_id: meerkat_core::SessionId::new(),
10262 usage: meerkat_core::Usage::default(),
10263 turns: 1,
10264 tool_calls: 0,
10265 terminal_cause_kind: None,
10266 structured_output: None,
10267 extraction_error: Some(meerkat_core::types::ExtractionError {
10268 last_output: "main output".to_string(),
10269 attempts: 3,
10270 reason: "invalid schema".to_string(),
10271 }),
10272 schema_warnings: None,
10273 skill_diagnostics: None,
10274 })),
10275 );
10276
10277 assert!(matches!(
10278 event,
10279 AgentEvent::InteractionFailed {
10280 interaction_id: actual_id,
10281 reason: meerkat_core::event::InteractionFailureReason::ExtractionFailed {
10282 ref last_output,
10283 attempts: 3,
10284 ref reason,
10285 },
10286 } if actual_id == interaction_id
10287 && last_output == "main output"
10288 && reason == "invalid schema"
10289 ));
10290 }
10291
10292 #[test]
10293 fn peer_turn_metadata_matches_live_interaction_complete_identity() {
10294 let interaction_id = InteractionId(Uuid::new_v4());
10295 let input = crate::input::Input::Peer(crate::input::PeerInput {
10296 directed_interaction_id: None,
10297 objective_id: None,
10298 system_prompts: Vec::new(),
10299 injected_context: Vec::new(),
10300 header: crate::input::InputHeader {
10301 id: meerkat_core::lifecycle::InputId::new(),
10302 timestamp: chrono::Utc::now(),
10303 source: crate::input::InputOrigin::Peer {
10304 peer_id: PEER_ID_RECEIVER.to_string(),
10305 display_identity: Some("keepalive-peer".to_string()),
10306 runtime_id: None,
10307 },
10308 durability: crate::input::InputDurability::Durable,
10309 visibility: crate::input::InputVisibility::default(),
10310 idempotency_key: None,
10311 supersession_key: None,
10312 correlation_id: Some(crate::CorrelationId::from_uuid(interaction_id.0)),
10313 },
10314 convention: Some(crate::input::PeerConvention::Message),
10315 content: "poke".to_string().into(),
10316 payload: None,
10317 handling_mode: None,
10318 sender_taint: None,
10319 });
10320 let semantics =
10321 crate::ingress_types::RuntimeInputSemantics::try_from_generated_admission(&input, true)
10322 .expect("peer message should admit as a runtime turn");
10323 let metadata = crate::runtime_loop::for_input(&input, semantics);
10324
10325 let event =
10326 interaction_terminal_event(interaction_id, CompletionOutcome::CompletedWithoutResult);
10327 let AgentEvent::InteractionComplete {
10328 interaction_id: live_interaction_id,
10329 ..
10330 } = event
10331 else {
10332 panic!("expected InteractionComplete");
10333 };
10334
10335 assert_eq!(
10336 metadata.transcript_identity.interaction_id,
10337 Some(live_interaction_id),
10338 "runtime transcript identity must use the same peer interaction id as the live terminal event"
10339 );
10340 }
10341
10342 #[test]
10343 fn callback_pending_maps_to_interaction_callback_pending_terminal_event() {
10344 let interaction_id = InteractionId(Uuid::new_v4());
10345 let event = interaction_terminal_event(
10346 interaction_id,
10347 CompletionOutcome::CallbackPending {
10348 tool_use_id: "call-1".to_string(),
10349 tool_name: "external_mock".to_string(),
10350 args: json!({ "value": "browser" }),
10351 },
10352 );
10353
10354 assert!(
10355 matches!(event, AgentEvent::InteractionCallbackPending { .. }),
10356 "expected callback-pending interaction event"
10357 );
10358 if let AgentEvent::InteractionCallbackPending {
10359 interaction_id: actual_id,
10360 tool_name,
10361 args,
10362 ..
10363 } = event
10364 {
10365 assert_eq!(actual_id, interaction_id);
10366 assert_eq!(tool_name, "external_mock");
10367 assert_eq!(args, json!({ "value": "browser" }));
10368 }
10369 }
10370
10371 #[tokio::test]
10372 async fn peer_lifecycle_added_does_not_change_comms_trust() {
10373 let runtime: Arc<dyn CommsRuntime> =
10374 Arc::new(meerkat_comms::CommsRuntime::inproc_only("receiver-added").unwrap());
10375 let peer = meerkat_comms::CommsRuntime::inproc_only("peer-added").unwrap();
10376 let peer_spec = trusted_peer_from_runtime("peer-added", &peer);
10377 let candidate = lifecycle_candidate(
10378 PeerInputClass::PeerLifecycleAdded,
10379 "mob.peer_added",
10380 json!({
10381 "peer": "peer-1",
10382 "peer_spec": peer_spec,
10383 }),
10384 );
10385
10386 let peers_before = runtime.peers().await;
10387 assert!(
10388 peers_before.is_empty(),
10389 "test runtime should start without trust"
10390 );
10391 let input =
10392 classified_interaction_to_runtime_input(&candidate, &LogicalRuntimeId::new("s-1"))
10393 .expect("lifecycle candidate should project to runtime input");
10394 assert!(
10395 matches!(input, Input::Peer(_)),
10396 "lifecycle candidate should still route as peer input"
10397 );
10398 let peers = runtime.peers().await;
10399 assert!(
10400 peers.is_empty(),
10401 "peer lifecycle add must not materialize comms trust before topology validation"
10402 );
10403 }
10404
10405 #[tokio::test]
10406 async fn peer_lifecycle_unwired_and_retired_do_not_revoke_comms_trust() {
10407 let runtime =
10408 Arc::new(meerkat_comms::CommsRuntime::inproc_only("receiver-removed").unwrap());
10409 let peer = meerkat_comms::CommsRuntime::inproc_only("peer-removed").unwrap();
10410 let peer_spec = trusted_peer_from_runtime("peer-removed", &peer);
10411 let dsl = test_projection_trust_dsl(runtime.as_ref());
10412 add_test_projection_trust_with_dsl(
10413 runtime.as_ref(),
10414 Arc::clone(&dsl),
10415 peer_spec.clone(),
10416 1,
10417 "trust peer",
10418 )
10419 .await;
10420
10421 let unwired = lifecycle_candidate(
10422 PeerInputClass::PeerLifecycleUnwired,
10423 "mob.peer_unwired",
10424 json!({
10425 "peer": "peer-1",
10426 "peer_spec": peer_spec.clone(),
10427 }),
10428 );
10429 let _ = classified_interaction_to_runtime_input(&unwired, &LogicalRuntimeId::new("s-1"))
10430 .expect("unwired lifecycle candidate should project");
10431 assert!(
10432 runtime
10433 .peers()
10434 .await
10435 .iter()
10436 .any(|entry| entry.name.as_str() == "peer-removed"),
10437 "peer lifecycle unwire must not revoke comms trust before topology validation"
10438 );
10439
10440 add_test_projection_trust_with_dsl(
10441 runtime.as_ref(),
10442 dsl,
10443 peer_spec.clone(),
10444 2,
10445 "refresh peer trust",
10446 )
10447 .await;
10448 let retired = lifecycle_candidate(
10449 PeerInputClass::PeerLifecycleRetired,
10450 "mob.peer_retired",
10451 json!({
10452 "peer": "peer-1",
10453 "peer_spec": peer_spec,
10454 }),
10455 );
10456 let _ = classified_interaction_to_runtime_input(&retired, &LogicalRuntimeId::new("s-1"))
10457 .expect("retired lifecycle candidate should project");
10458 assert!(
10459 runtime
10460 .peers()
10461 .await
10462 .iter()
10463 .any(|entry| entry.name.as_str() == "peer-removed"),
10464 "peer lifecycle retire must not revoke comms trust before topology validation"
10465 );
10466 }
10467
10468 #[test]
10469 fn bridge_capabilities_report_canonical_protocol_versions() {
10470 let capabilities = bridge_capabilities(
10471 meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion::V4,
10472 );
10473 assert_eq!(
10474 capabilities.current_protocol_version,
10475 supervisor_bridge_current_protocol_version()
10476 );
10477 assert_eq!(
10478 capabilities.default_protocol_version,
10479 supervisor_bridge_default_protocol_version()
10480 );
10481 assert_eq!(
10482 capabilities.supported_protocol_versions,
10483 supervisor_bridge_supported_protocol_versions()
10484 );
10485 assert!(
10486 capabilities.hard_cancel_member,
10487 "phase 6 serves HardCancelMember: the bridge MUST advertise it — \
10488 the controlling side capability-gates dispatch on this fact \
10489 (ADJ-P6-5), so a false report would typed-reject every hard \
10490 cancel without bridge traffic"
10491 );
10492 assert!(
10493 capabilities.tracked_input_cancel,
10494 "V4 serves exact durable tracked-input cancellation and must advertise it"
10495 );
10496 }
10497
10498 #[test]
10499 fn v3_bind_capabilities_omit_v4_extensions_and_hard_cancel() {
10500 let capabilities = bridge_capabilities(
10501 meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion::V3,
10502 );
10503 assert_eq!(
10504 capabilities.current_protocol_version,
10505 meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion::V3
10506 );
10507 assert!(!capabilities.hard_cancel_member);
10508 assert!(!capabilities.tracked_input_cancel);
10509 assert_eq!(
10510 capabilities.supported_protocol_versions,
10511 vec![
10512 meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion::V2,
10513 meerkat_contracts::wire::supervisor_bridge::BridgeProtocolVersion::V3,
10514 ]
10515 );
10516 let encoded = serde_json::to_value(&capabilities).expect("serialize V3 capabilities");
10517 for v4_field in [
10518 "durable_sessions",
10519 "autonomous_members",
10520 "tracked_input_cancel",
10521 "memory_store",
10522 "mcp",
10523 "engine_version",
10524 "approval_forwarding",
10525 "resolvable_providers",
10526 ] {
10527 assert!(
10528 encoded.get(v4_field).is_none(),
10529 "V3 bind response leaked V4 capability field {v4_field}: {encoded}"
10530 );
10531 }
10532 }
10533
10534 #[tokio::test]
10535 async fn validate_bind_request_rejects_missing_or_wrong_bootstrap_token() {
10536 let runtime: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
10537 PEER_ID_RECEIVER,
10538 "inproc://receiver",
10539 Some("expected-token"),
10540 ));
10541 let (adapter, session_id) = bind_material_adapter().await;
10542 let supervisor = supervisor_bridge_spec();
10543 let payload = meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
10544 supervisor: supervisor.clone(),
10545 epoch: 0,
10546 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
10547 expected_peer_id: PEER_ID_RECEIVER.to_string(),
10548 expected_address: runtime.advertised_address().unwrap(),
10549 bootstrap_token: "wrong-token".into(),
10550 };
10551
10552 let (cause, error) = validate_bind_request(
10553 &adapter,
10554 &session_id,
10555 &runtime,
10556 &bridge_sender_fact(&supervisor.peer_id),
10557 &payload,
10558 )
10559 .await
10560 .expect_err("bind must reject incorrect bootstrap token");
10561 assert_eq!(cause, BridgeRejectionCause::InvalidBootstrapToken);
10562 assert!(
10563 error.contains("invalid bootstrap token"),
10564 "bind rejection should explain the bootstrap proof failure, got: {error}"
10565 );
10566 }
10567
10568 #[tokio::test]
10569 async fn validate_bind_request_accepts_matching_bootstrap_token() {
10570 let runtime: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
10571 PEER_ID_RECEIVER,
10572 "inproc://receiver",
10573 Some("expected-token"),
10574 ));
10575 let (adapter, session_id) = bind_material_adapter().await;
10576 let supervisor = supervisor_bridge_spec();
10577 let payload = meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
10578 supervisor: supervisor.clone(),
10579 epoch: 0,
10580 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
10581 expected_peer_id: PEER_ID_RECEIVER.to_string(),
10582 expected_address: runtime.advertised_address().unwrap(),
10583 bootstrap_token: "expected-token".into(),
10584 };
10585
10586 let (authorized, advertised_address) = validate_bind_request(
10587 &adapter,
10588 &session_id,
10589 &runtime,
10590 &bridge_sender_fact(&supervisor.peer_id),
10591 &payload,
10592 )
10593 .await
10594 .expect("bind should accept the configured bootstrap token");
10595 assert_eq!(authorized.peer_id.as_str(), supervisor.peer_id);
10596 assert_eq!(advertised_address, runtime.advertised_address().unwrap());
10597 }
10598
10599 #[test]
10600 fn bridge_delivery_payload_preserves_explicit_queue_handling_mode() {
10601 let stable_input_id = Uuid::new_v4();
10602 let input = peer_input_from_delivery_payload(
10603 &SessionId::new(),
10604 PeerId::parse(PEER_ID_SUPERVISOR).expect("valid supervisor peer id"),
10605 BridgeDeliveryPayload {
10606 objective_id: None,
10607 system_prompt: None,
10608 injected_context: Vec::new(),
10609 transient_turn_context: None,
10610 supervisor: supervisor_bridge_spec(),
10611 epoch: 1,
10612 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
10613 input_id: stable_input_id.to_string(),
10614 transcript_interaction_id: None,
10615 content: meerkat_core::types::ContentInput::Text("queued follow-up".to_string()),
10616 handling_mode: HandlingMode::Queue,
10617 expected_member: None,
10618 turn: None,
10619 outcome_tracking: None,
10620 },
10621 )
10622 .expect("UUID payload input id lowers");
10623
10624 let Input::Peer(peer) = &input else {
10625 panic!("bridge delivery must project to peer input");
10626 };
10627 assert_eq!(
10628 peer.handling_mode,
10629 Some(HandlingMode::Queue),
10630 "bridge delivery explicit queue must survive into MeerkatMachine admission"
10631 );
10632 assert_eq!(peer.header.id.0, stable_input_id);
10633 assert_eq!(
10634 peer.header.correlation_id.as_ref().map(|id| id.0),
10635 Some(stable_input_id),
10636 "payload input_id is the stable terminal correlation, not the transient envelope id"
10637 );
10638 assert_eq!(
10639 peer.directed_interaction_id, None,
10640 "peer-only legacy delivery must not mint terminal-publication authority"
10641 );
10642 assert_eq!(
10643 crate::input::validated_directed_interaction_id(&input)
10644 .expect("legacy peer shape validates"),
10645 None
10646 );
10647 let semantics =
10648 crate::ingress_types::RuntimeInputSemantics::try_from_generated_admission(&input, true)
10649 .expect("legacy peer admission semantics");
10650 assert!(
10651 crate::runtime_loop::for_input(&input, semantics)
10652 .directed_interaction_ids
10653 .is_empty(),
10654 "correlation alone must not opt legacy peer traffic into directed terminals"
10655 );
10656 }
10657
10658 #[test]
10659 fn bridge_delivery_transient_context_preserves_runtime_owned_metadata() {
10660 let stable_input_id = Uuid::new_v4();
10661 let objective_id = meerkat_core::interaction::ObjectiveId::new();
10662 let transient_context =
10663 meerkat_core::lifecycle::run_primitive::TurnRequestContext::new("caller context")
10664 .expect("non-empty transient context");
10665 let input = peer_input_from_delivery_payload(
10666 &SessionId::new(),
10667 PeerId::parse(PEER_ID_SUPERVISOR).expect("valid supervisor peer id"),
10668 BridgeDeliveryPayload {
10669 objective_id: Some(objective_id),
10670 system_prompt: Some("updated member instructions".to_string()),
10671 injected_context: Vec::new(),
10672 transient_turn_context: Some(transient_context.clone()),
10673 supervisor: supervisor_bridge_spec(),
10674 epoch: 1,
10675 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
10676 input_id: stable_input_id.to_string(),
10677 transcript_interaction_id: None,
10678 content: meerkat_core::types::ContentInput::Text("live follow-up".to_string()),
10679 handling_mode: HandlingMode::Steer,
10680 expected_member: None,
10681 turn: None,
10682 outcome_tracking: None,
10683 },
10684 )
10685 .expect("transient-context bridge delivery lowers");
10686
10687 let Input::FlowStep(flow_step) = &input else {
10688 panic!("transient-context delivery must use the flow-step carrier");
10689 };
10690 let metadata = flow_step
10691 .turn_metadata
10692 .as_ref()
10693 .expect("flow-step carrier must preserve turn metadata");
10694 assert_eq!(metadata.handling_mode, Some(HandlingMode::Steer));
10695 assert_eq!(
10696 metadata.system_prompts,
10697 ["updated member instructions"],
10698 "supervisor-authored System content must survive bridge lowering"
10699 );
10700 assert_eq!(
10701 metadata.transient_turn_context.as_ref(),
10702 Some(&transient_context)
10703 );
10704 assert_eq!(
10705 metadata.transcript_identity.objective_id,
10706 Some(objective_id)
10707 );
10708
10709 let semantics =
10710 crate::ingress_types::RuntimeInputSemantics::try_from_generated_admission(&input, true)
10711 .expect("flow-step admission semantics");
10712 let runtime_metadata = crate::runtime_loop::for_input(&input, semantics);
10713 assert_eq!(
10714 runtime_metadata.transcript_identity.interaction_id,
10715 Some(meerkat_core::interaction::InteractionId(stable_input_id)),
10716 "the runtime constructor must mint the transient turn's durable interaction identity"
10717 );
10718 }
10719
10720 #[test]
10721 fn explicitly_tracked_plain_bridge_delivery_mints_exact_directed_interaction() {
10722 let session_id = SessionId::new();
10723 let stable_input_id = Uuid::new_v4();
10724 let objective_id = meerkat_core::interaction::ObjectiveId::new();
10725 let payload = BridgeDeliveryPayload {
10726 objective_id: Some(objective_id),
10727 system_prompt: Some("updated member instructions".to_string()),
10728 injected_context: vec![meerkat_core::types::ContentInput::Text(
10729 "tracked ambient context".to_string(),
10730 )],
10731 transient_turn_context: None,
10732 supervisor: supervisor_bridge_spec(),
10733 epoch: 1,
10734 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
10735 input_id: stable_input_id.to_string(),
10736 transcript_interaction_id: Some(stable_input_id.to_string()),
10737 content: meerkat_core::types::ContentInput::Text("placed work".to_string()),
10738 handling_mode: HandlingMode::Queue,
10739 expected_member: Some(BridgeMemberIncarnation {
10740 mob_id: "mob".to_string(),
10741 agent_identity: "worker".to_string(),
10742 host_id: "host".to_string(),
10743 binding_generation: 1,
10744 member_session_id: session_id.to_string(),
10745 generation: 1,
10746 fence_token: 11,
10747 }),
10748 turn: None,
10749 outcome_tracking: Some(BridgeOutcomeTracking::Interaction),
10750 };
10751 assert!(delivery_requires_tracked_turn_custody(&payload));
10752 let input = peer_input_from_delivery_payload(
10753 &session_id,
10754 PeerId::parse(PEER_ID_SUPERVISOR).expect("valid supervisor peer id"),
10755 payload,
10756 )
10757 .expect("placed UUID payload lowers");
10758
10759 let expected = meerkat_core::interaction::InteractionId(stable_input_id);
10760 let Input::Peer(peer) = &input else {
10761 panic!("placed bridge delivery must project to peer input");
10762 };
10763 assert_eq!(peer.directed_interaction_id, Some(expected));
10764 assert_eq!(peer.objective_id, Some(objective_id));
10765 assert_eq!(
10766 peer.system_prompts,
10767 ["updated member instructions"],
10768 "supervisor-authored System content must survive bridge lowering"
10769 );
10770 assert_eq!(
10771 peer.injected_context,
10772 vec![meerkat_core::types::ContentInput::Text(
10773 "tracked ambient context".to_string()
10774 )]
10775 );
10776 assert_eq!(
10777 crate::input::validated_directed_interaction_id(&input)
10778 .expect("placed peer shape validates"),
10779 Some(expected)
10780 );
10781 let semantics =
10782 crate::ingress_types::RuntimeInputSemantics::try_from_generated_admission(&input, true)
10783 .expect("placed peer admission semantics");
10784 let turn_metadata = crate::runtime_loop::for_input(&input, semantics);
10785 assert_eq!(turn_metadata.directed_interaction_ids, vec![expected]);
10786 assert_eq!(
10787 turn_metadata.system_prompts,
10788 ["updated member instructions"],
10789 "the runtime boundary must preserve the exact per-turn System row"
10790 );
10791 }
10792
10793 #[test]
10794 fn placed_delivery_without_explicit_tracking_remains_untracked() {
10795 let session_id = SessionId::new();
10796 let stable_input_id = Uuid::new_v4();
10797 let transcript_interaction_id = Uuid::new_v4();
10798 let payload = BridgeDeliveryPayload {
10799 objective_id: None,
10800 system_prompt: None,
10801 injected_context: Vec::new(),
10802 transient_turn_context: None,
10803 supervisor: supervisor_bridge_spec(),
10804 epoch: 1,
10805 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
10806 input_id: stable_input_id.to_string(),
10807 transcript_interaction_id: Some(transcript_interaction_id.to_string()),
10808 content: meerkat_core::types::ContentInput::Text("ordinary placed work".to_string()),
10809 handling_mode: HandlingMode::Queue,
10810 expected_member: Some(BridgeMemberIncarnation {
10811 mob_id: "mob".to_string(),
10812 agent_identity: "worker".to_string(),
10813 host_id: "host".to_string(),
10814 binding_generation: 1,
10815 member_session_id: session_id.to_string(),
10816 generation: 1,
10817 fence_token: 11,
10818 }),
10819 turn: None,
10820 outcome_tracking: None,
10821 };
10822 assert!(!delivery_requires_tracked_turn_custody(&payload));
10823 validate_delivery_tracking_request(&payload)
10824 .expect("untracked placed admission accepts independent transcript identity");
10825 let mut unscoped = payload.clone();
10826 unscoped.expected_member = None;
10827 assert!(
10828 validate_delivery_tracking_request(&unscoped)
10829 .expect_err("transcript identity without placed residency must reject")
10830 .contains("exact placed-member incarnation")
10831 );
10832 let mut under_versioned = payload.clone();
10833 under_versioned.protocol_version = BridgeProtocolVersion::V3;
10834 assert!(
10835 validate_delivery_tracking_request(&under_versioned)
10836 .expect_err("placed transcript identity requires V4")
10837 .contains("V4")
10838 );
10839 let input = peer_input_from_delivery_payload(
10840 &session_id,
10841 PeerId::parse(PEER_ID_SUPERVISOR).expect("valid supervisor peer id"),
10842 payload,
10843 )
10844 .expect("ordinary placed payload lowers");
10845 let Input::Peer(peer) = &input else {
10846 panic!("ordinary placed bridge delivery must project to peer input");
10847 };
10848 assert_eq!(peer.directed_interaction_id, None);
10849 let expected = meerkat_core::interaction::InteractionId(transcript_interaction_id);
10850 let semantics =
10851 crate::ingress_types::RuntimeInputSemantics::try_from_generated_admission(&input, true)
10852 .expect("ordinary placed peer admission semantics");
10853 let metadata = crate::runtime_loop::for_input(&input, semantics);
10854 assert_eq!(
10855 metadata.transcript_identity.interaction_id,
10856 Some(expected),
10857 "the independent transcript carrier survives without terminal tracking"
10858 );
10859 assert!(
10860 metadata.directed_interaction_ids.is_empty(),
10861 "IngressAccepted must not acquire terminal-publication authority"
10862 );
10863 assert_eq!(
10864 crate::input::validated_directed_interaction_id(&input)
10865 .expect("ordinary placed peer shape validates"),
10866 None
10867 );
10868 }
10869
10870 #[test]
10871 fn placed_admission_without_caller_transcript_id_keeps_identity_absent() {
10872 let session_id = SessionId::new();
10873 let input = peer_input_from_delivery_payload(
10874 &session_id,
10875 PeerId::parse(PEER_ID_SUPERVISOR).expect("valid supervisor peer id"),
10876 BridgeDeliveryPayload {
10877 objective_id: None,
10878 system_prompt: None,
10879 injected_context: Vec::new(),
10880 transient_turn_context: None,
10881 supervisor: supervisor_bridge_spec(),
10882 epoch: 1,
10883 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
10884 input_id: Uuid::new_v4().to_string(),
10885 transcript_interaction_id: None,
10886 content: meerkat_core::types::ContentInput::Text(
10887 "ordinary placed work".to_string(),
10888 ),
10889 handling_mode: HandlingMode::Queue,
10890 expected_member: Some(BridgeMemberIncarnation {
10891 mob_id: "mob".to_string(),
10892 agent_identity: "worker".to_string(),
10893 host_id: "host".to_string(),
10894 binding_generation: 1,
10895 member_session_id: session_id.to_string(),
10896 generation: 1,
10897 fence_token: 11,
10898 }),
10899 turn: None,
10900 outcome_tracking: None,
10901 },
10902 )
10903 .expect("ordinary placed payload lowers");
10904 assert!(input.header().correlation_id.is_none());
10905 let semantics =
10906 crate::ingress_types::RuntimeInputSemantics::try_from_generated_admission(&input, true)
10907 .expect("ordinary placed peer admission semantics");
10908 assert!(
10909 crate::runtime_loop::for_input(&input, semantics)
10910 .transcript_identity
10911 .interaction_id
10912 .is_none(),
10913 "absent caller identity must remain absent for runtime-side minting"
10914 );
10915 }
10916
10917 #[test]
10918 fn tracked_interaction_validation_is_v4_exact_member_and_mutually_exclusive() {
10919 let session_id = SessionId::new();
10920 let mut payload = BridgeDeliveryPayload {
10921 objective_id: None,
10922 system_prompt: None,
10923 injected_context: Vec::new(),
10924 transient_turn_context: None,
10925 supervisor: supervisor_bridge_spec(),
10926 epoch: 1,
10927 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
10928 input_id: Uuid::new_v4().to_string(),
10929 transcript_interaction_id: None,
10930 content: meerkat_core::types::ContentInput::Text("tracked work".to_string()),
10931 handling_mode: HandlingMode::Queue,
10932 expected_member: Some(BridgeMemberIncarnation {
10933 mob_id: "mob".to_string(),
10934 agent_identity: "worker".to_string(),
10935 host_id: "host".to_string(),
10936 binding_generation: 1,
10937 member_session_id: session_id.to_string(),
10938 generation: 1,
10939 fence_token: 11,
10940 }),
10941 turn: None,
10942 outcome_tracking: Some(BridgeOutcomeTracking::Interaction),
10943 };
10944 payload.transcript_interaction_id = Some(payload.input_id.clone());
10945 validate_delivery_tracking_request(&payload).expect("valid tracked interaction");
10946
10947 payload.transcript_interaction_id = Some(Uuid::new_v4().to_string());
10948 assert!(
10949 validate_delivery_tracking_request(&payload)
10950 .expect_err("tracked terminal identity must not split from its sidecar key")
10951 .contains("equal input_id")
10952 );
10953 payload.transcript_interaction_id =
10954 Some("AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA".to_string());
10955 assert!(
10956 validate_delivery_tracking_request(&payload)
10957 .expect_err("non-canonical transcript UUID must reject")
10958 .contains("canonical non-nil UUID")
10959 );
10960 payload.transcript_interaction_id = Some(payload.input_id.clone());
10961
10962 let canonical_input_id = payload.input_id.clone();
10963 payload.input_id = Uuid::nil().to_string();
10964 payload.transcript_interaction_id = Some(payload.input_id.clone());
10965 assert!(
10966 validate_delivery_tracking_request(&payload)
10967 .expect_err("tracked peer nil UUID must reject")
10968 .contains("nil UUID")
10969 );
10970 assert!(
10971 peer_input_from_delivery_payload(
10972 &session_id,
10973 PeerId::parse(PEER_ID_SUPERVISOR).expect("valid supervisor peer id"),
10974 payload.clone(),
10975 )
10976 .expect_err("tracked peer lowering must independently reject nil UUID")
10977 .contains("nil UUID")
10978 );
10979 payload.input_id = canonical_input_id;
10980 payload.transcript_interaction_id = Some(payload.input_id.clone());
10981
10982 payload.protocol_version = BridgeProtocolVersion::V3;
10983 assert!(
10984 validate_delivery_tracking_request(&payload)
10985 .expect_err("pre-V4 tracking marker must reject")
10986 .contains("V4")
10987 );
10988 payload.protocol_version = SUPERVISOR_BRIDGE_PROTOCOL_VERSION;
10989 payload.expected_member = None;
10990 assert!(
10991 validate_delivery_tracking_request(&payload)
10992 .expect_err("tracking without exact residency must reject")
10993 .contains("exact placed-member incarnation")
10994 );
10995 payload.expected_member = Some(BridgeMemberIncarnation {
10996 mob_id: "mob".to_string(),
10997 agent_identity: "worker".to_string(),
10998 host_id: "host".to_string(),
10999 binding_generation: 1,
11000 member_session_id: session_id.to_string(),
11001 generation: 1,
11002 fence_token: 11,
11003 });
11004 payload.turn = Some(
11005 meerkat_contracts::wire::supervisor_bridge::BridgeTurnDirective {
11006 correlation: meerkat_contracts::wire::supervisor_bridge::BridgeTurnCorrelation {
11007 run_id: "run".to_string(),
11008 step_id: "step".to_string(),
11009 },
11010 tool_overlay: None,
11011 },
11012 );
11013 assert!(
11014 validate_delivery_tracking_request(&payload)
11015 .expect_err("turn and interaction marker must reject")
11016 .contains("mutually exclusive")
11017 );
11018
11019 payload.outcome_tracking = None;
11020 payload.transcript_interaction_id = None;
11021 payload.input_id = Uuid::nil().to_string();
11022 assert!(
11023 validate_delivery_tracking_request(&payload)
11024 .expect_err("tracked flow-step nil UUID must reject")
11025 .contains("nil UUID")
11026 );
11027 }
11028
11029 #[test]
11030 fn forged_tracked_peer_interaction_identity_is_rejected() {
11031 let session_id = SessionId::new();
11032 let tracked_id = Uuid::new_v4();
11033 let mut input = peer_input_from_delivery_payload(
11034 &session_id,
11035 PeerId::parse(PEER_ID_SUPERVISOR).expect("valid supervisor peer id"),
11036 BridgeDeliveryPayload {
11037 objective_id: None,
11038 system_prompt: None,
11039 injected_context: Vec::new(),
11040 transient_turn_context: None,
11041 supervisor: supervisor_bridge_spec(),
11042 epoch: 1,
11043 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
11044 input_id: tracked_id.to_string(),
11045 transcript_interaction_id: Some(tracked_id.to_string()),
11046 content: meerkat_core::types::ContentInput::Text("placed work".to_string()),
11047 handling_mode: HandlingMode::Queue,
11048 expected_member: Some(BridgeMemberIncarnation {
11049 mob_id: "mob".to_string(),
11050 agent_identity: "worker".to_string(),
11051 host_id: "host".to_string(),
11052 binding_generation: 1,
11053 member_session_id: session_id.to_string(),
11054 generation: 1,
11055 fence_token: 11,
11056 }),
11057 turn: None,
11058 outcome_tracking: Some(BridgeOutcomeTracking::Interaction),
11059 },
11060 )
11061 .expect("placed UUID payload lowers");
11062 let Input::Peer(peer) = &mut input else {
11063 panic!("placed bridge delivery must project to peer input");
11064 };
11065 peer.directed_interaction_id =
11066 Some(meerkat_core::interaction::InteractionId(Uuid::new_v4()));
11067
11068 assert!(crate::input::validated_directed_interaction_id(&input).is_err());
11069 }
11070
11071 #[test]
11072 fn bridge_delivery_payload_rejects_non_uuid_stable_input_id() {
11073 let lowered = peer_input_from_delivery_payload(
11074 &SessionId::new(),
11075 PeerId::parse(PEER_ID_SUPERVISOR).expect("valid supervisor peer id"),
11076 BridgeDeliveryPayload {
11077 objective_id: None,
11078 system_prompt: None,
11079 injected_context: Vec::new(),
11080 transient_turn_context: None,
11081 supervisor: supervisor_bridge_spec(),
11082 epoch: 1,
11083 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
11084 input_id: "transport-envelope-alias".to_string(),
11085 transcript_interaction_id: None,
11086 content: meerkat_core::types::ContentInput::Text("work".to_string()),
11087 handling_mode: HandlingMode::Queue,
11088 expected_member: None,
11089 turn: None,
11090 outcome_tracking: None,
11091 },
11092 );
11093 assert!(lowered.is_err());
11094 }
11095
11096 #[test]
11101 fn bridge_delivery_payload_injected_context_projects_before_peer_append() {
11102 let objective_id = meerkat_core::interaction::ObjectiveId::new();
11103 let input = peer_input_from_delivery_payload(
11104 &SessionId::new(),
11105 PeerId::parse(PEER_ID_SUPERVISOR).expect("valid supervisor peer id"),
11106 BridgeDeliveryPayload {
11107 objective_id: Some(objective_id),
11108 system_prompt: None,
11109 injected_context: vec![
11110 meerkat_core::types::ContentInput::Text("ambient alpha".to_string()),
11111 meerkat_core::types::ContentInput::Text("ambient beta".to_string()),
11112 ],
11113 transient_turn_context: None,
11114 supervisor: supervisor_bridge_spec(),
11115 epoch: 1,
11116 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
11117 input_id: Uuid::new_v4().to_string(),
11118 transcript_interaction_id: None,
11119 content: meerkat_core::types::ContentInput::Text("work content".to_string()),
11120 handling_mode: HandlingMode::Queue,
11121 expected_member: None,
11122 turn: None,
11123 outcome_tracking: None,
11124 },
11125 )
11126 .expect("UUID payload input id lowers");
11127
11128 let Input::Peer(peer) = &input else {
11129 panic!("bridge delivery must project to peer input");
11130 };
11131 assert_eq!(
11132 peer.injected_context,
11133 vec![
11134 meerkat_core::types::ContentInput::Text("ambient alpha".to_string()),
11135 meerkat_core::types::ContentInput::Text("ambient beta".to_string()),
11136 ],
11137 );
11138 assert_eq!(peer.objective_id, Some(objective_id));
11139
11140 let projection = crate::input::runtime_input_projection_for_machine_batch(&input);
11141 assert_eq!(
11142 projection
11143 .injected_context_appends
11144 .iter()
11145 .map(|append| (append.role, append.content.render_text()))
11146 .collect::<Vec<_>>(),
11147 vec![
11148 (
11149 meerkat_core::lifecycle::run_primitive::ConversationAppendRole::InjectedContext,
11150 "ambient alpha".to_string()
11151 ),
11152 (
11153 meerkat_core::lifecycle::run_primitive::ConversationAppendRole::InjectedContext,
11154 "ambient beta".to_string()
11155 ),
11156 ],
11157 );
11158 assert!(
11159 projection.append.is_some(),
11160 "peer work append must survive alongside injected context"
11161 );
11162 }
11163
11164 #[test]
11165 fn bridge_delivery_handler_does_not_wait_for_completion_inline() {
11166 let source = include_str!("comms_drain.rs");
11167 let deliver_start = source
11168 .find("BridgeCommand::DeliverMemberInput")
11169 .expect("deliver command branch exists");
11170 let deliver_end = source[deliver_start..]
11171 .find("fn bridge_runtime_state")
11172 .map(|offset| deliver_start + offset)
11173 .expect("bridge runtime state helper follows command match");
11174 let deliver_source = &source[deliver_start..deliver_end];
11175 assert!(
11176 !deliver_source.contains(".wait().await"),
11177 "bridge delivery admission must not block the comms drain waiting for completion"
11178 );
11179 }
11180
11181 #[tokio::test]
11182 async fn validate_bind_request_rejects_same_display_name_different_canonical_sender() {
11183 let runtime: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
11184 PEER_ID_RECEIVER,
11185 "inproc://receiver",
11186 Some("expected-token"),
11187 ));
11188 let (adapter, session_id) = bind_material_adapter().await;
11189 let supervisor = supervisor_bridge_spec();
11190 let payload = meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
11191 supervisor: supervisor.clone(),
11192 epoch: 0,
11193 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
11194 expected_peer_id: PEER_ID_RECEIVER.to_string(),
11195 expected_address: runtime.advertised_address().unwrap(),
11196 bootstrap_token: "expected-token".into(),
11197 };
11198 let attacker_peer_id =
11199 PeerId::parse(PEER_ID_OLD_SUPERVISOR).expect("valid attacker peer id");
11200 let sender = bridge_sender_fact_with_display(attacker_peer_id, &supervisor.name);
11201
11202 let (cause, error) =
11203 validate_bind_request(&adapter, &session_id, &runtime, &sender, &payload)
11204 .await
11205 .expect_err("same display name with a different canonical sender must reject");
11206 assert_eq!(cause, BridgeRejectionCause::SenderMismatch);
11207 assert!(
11208 error.contains("does not match supervisor"),
11209 "bind rejection should explain the sender mismatch, got: {error}"
11210 );
11211 }
11212
11213 #[tokio::test]
11214 async fn validate_bind_request_accepts_pubkey_sender_with_canonical_supervisor_peer_id() {
11215 let runtime: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
11216 PEER_ID_RECEIVER,
11217 "inproc://receiver",
11218 Some("expected-token"),
11219 ));
11220 let (adapter, session_id) = bind_material_adapter().await;
11221 let supervisor_key = meerkat_comms::Keypair::generate();
11222 let supervisor_pubkey = supervisor_key.public_key();
11223 let supervisor = BridgePeerSpec {
11224 name: "mob/__mob_supervisor__".to_string(),
11225 peer_id: supervisor_pubkey.to_peer_id().as_str(),
11226 address: "inproc://mob/__mob_supervisor__".to_string(),
11227 pubkey: *supervisor_pubkey.as_bytes(),
11228 };
11229 let payload = meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
11230 supervisor: supervisor.clone(),
11231 epoch: 0,
11232 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
11233 expected_peer_id: PEER_ID_RECEIVER.to_string(),
11234 expected_address: runtime.advertised_address().unwrap(),
11235 bootstrap_token: "expected-token".into(),
11236 };
11237
11238 let (authorized, advertised_address) = validate_bind_request(
11239 &adapter,
11240 &session_id,
11241 &runtime,
11242 &bridge_sender_fact(&supervisor_pubkey.to_pubkey_string()),
11243 &payload,
11244 )
11245 .await
11246 .expect("bind should accept raw transport sender when payload carries pubkey");
11247 assert_eq!(authorized.peer_id.as_str(), supervisor.peer_id);
11248 assert_eq!(authorized.pubkey, supervisor.pubkey);
11249 assert_eq!(advertised_address, runtime.advertised_address().unwrap());
11250 }
11251
11252 #[tokio::test]
11253 async fn validate_bind_request_returns_runtime_advertised_address() {
11254 let runtime: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
11255 PEER_ID_RECEIVER,
11256 &format!(
11257 "inproc://receiver-real?{SUPERVISOR_BRIDGE_BOOTSTRAP_TOKEN_PARAM}=expected-token"
11258 ),
11259 Some("expected-token"),
11260 ));
11261 let (adapter, session_id) = bind_material_adapter().await;
11262 let supervisor = supervisor_bridge_spec();
11263 let payload = meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
11264 supervisor: supervisor.clone(),
11265 epoch: 0,
11266 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
11267 expected_peer_id: PEER_ID_RECEIVER.to_string(),
11268 expected_address: "inproc://receiver-real".to_string(),
11269 bootstrap_token: "expected-token".into(),
11270 };
11271
11272 let (_, advertised_address) = validate_bind_request(
11273 &adapter,
11274 &session_id,
11275 &runtime,
11276 &bridge_sender_fact(&supervisor.peer_id),
11277 &payload,
11278 )
11279 .await
11280 .expect("bind should canonicalize to the callee's advertised address");
11281 assert_eq!(advertised_address, runtime.advertised_address().unwrap());
11282 }
11283
11284 #[tokio::test]
11285 async fn validate_bind_request_rejects_mismatched_expected_address() {
11286 let runtime: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
11287 PEER_ID_RECEIVER,
11288 "inproc://receiver-real",
11289 Some("expected-token"),
11290 ));
11291 let (adapter, session_id) = bind_material_adapter().await;
11292 let supervisor = supervisor_bridge_spec();
11293 let payload = meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
11294 supervisor: supervisor.clone(),
11295 epoch: 0,
11296 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
11297 expected_peer_id: PEER_ID_RECEIVER.to_string(),
11298 expected_address: "inproc://receiver-stale".to_string(),
11299 bootstrap_token: "expected-token".into(),
11300 };
11301
11302 let (cause, error) = validate_bind_request(
11303 &adapter,
11304 &session_id,
11305 &runtime,
11306 &bridge_sender_fact(&supervisor.peer_id),
11307 &payload,
11308 )
11309 .await
11310 .expect_err("bind should reject mismatched expected addresses");
11311 assert_eq!(cause, BridgeRejectionCause::AddressMismatch);
11312 assert!(
11313 error.contains("bind address mismatch"),
11314 "bind rejection should explain the address mismatch, got: {error}"
11315 );
11316 }
11317
11318 #[tokio::test]
11319 async fn bridge_handler_rejects_unsupported_protocol_version_before_bind_validation() {
11320 let sent: Arc<tokio::sync::Mutex<Vec<CommsCommand>>> =
11321 Arc::new(tokio::sync::Mutex::new(Vec::new()));
11322 let runtime: Arc<dyn CommsRuntime> = Arc::new(CapturingRuntime {
11323 peer_id: PEER_ID_RECEIVER.to_string(),
11324 advertised_address: Some("inproc://receiver".to_string()),
11325 bootstrap_token: Some("expected-token".to_string()),
11326 inbox_notify: Arc::new(tokio::sync::Notify::new()),
11327 sent: sent.clone(),
11328 });
11329 let adapter = Arc::new(MeerkatMachine::ephemeral());
11330 let session_id = SessionId::new();
11331 adapter
11332 .register_session(session_id.clone())
11333 .await
11334 .expect("register session");
11335 let params = json!({
11336 "command": "bind_member",
11337 "supervisor": {
11338 "name": "mob/__mob_supervisor__",
11339 "peer_id": PEER_ID_SUPERVISOR,
11340 "address": "inproc://mob/__mob_supervisor__",
11341 "pubkey": vec![0u8; 32],
11342 },
11343 "epoch": 0,
11344 "protocol_version": 999,
11345 "expected_peer_id": PEER_ID_RECEIVER,
11346 "expected_address": "inproc://receiver",
11347 "bootstrap_token": "expected-token",
11348 });
11349 let interaction_id = InteractionId(Uuid::new_v4());
11350 let candidate = PeerInputCandidate {
11351 interaction: InboxInteraction {
11352 objective_id: None,
11353 sender_taint: None,
11354 id: interaction_id,
11355 from_route: None,
11356 from: PEER_ID_SUPERVISOR.to_string(),
11357 content: InteractionContent::Request {
11358 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
11359 params,
11360 blocks: None,
11361 },
11362 rendered_text: String::new(),
11363 handling_mode: HandlingMode::Queue,
11364 render_metadata: None,
11365 },
11366 ingress: bridge_sender_fact_with_id(interaction_id, PEER_ID_SUPERVISOR),
11367 lifecycle_peer: None,
11368 response_terminality: None,
11369 };
11370
11371 assert!(
11372 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate).await,
11373 "bridge handler must own malformed supervisor bridge commands"
11374 );
11375
11376 let (result, status) = sent
11377 .lock()
11378 .await
11379 .iter()
11380 .find_map(|cmd| match cmd {
11381 CommsCommand::PeerResponse { result, status, .. } => {
11382 Some((result.clone(), *status))
11383 }
11384 _ => None,
11385 })
11386 .expect("handler must send a typed rejection response");
11387 assert!(matches!(
11388 status,
11389 meerkat_core::interaction::ResponseStatus::Failed
11390 ));
11391 let reply: BridgeReply = serde_json::from_value(result).expect("typed bridge reply");
11392 match reply {
11393 BridgeReply::Rejected { cause, reason } => {
11394 assert_eq!(cause, BridgeRejectionCause::UnsupportedProtocolVersion);
11395 assert!(
11396 reason.contains("unsupported supervisor bridge protocol version"),
11397 "rejection should explain the typed version failure, got: {reason}"
11398 );
11399 }
11400 other => unreachable!("expected Rejected reply, got {other:?}"),
11401 }
11402 }
11403
11404 #[tokio::test]
11405 async fn validate_bind_request_rejects_invalid_supervisor_peer_name() {
11406 let runtime: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
11407 PEER_ID_RECEIVER,
11408 "inproc://receiver",
11409 Some("expected-token"),
11410 ));
11411 let (adapter, session_id) = bind_material_adapter().await;
11412 let mut supervisor = supervisor_bridge_spec();
11413 supervisor.name = String::new();
11414 let payload = meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
11415 supervisor,
11416 epoch: 0,
11417 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
11418 expected_peer_id: PEER_ID_RECEIVER.to_string(),
11419 expected_address: runtime.advertised_address().unwrap(),
11420 bootstrap_token: "expected-token".into(),
11421 };
11422
11423 let (cause, error) = validate_bind_request(
11424 &adapter,
11425 &session_id,
11426 &runtime,
11427 &bridge_sender_fact(&payload.supervisor.peer_id),
11428 &payload,
11429 )
11430 .await
11431 .expect_err("bind should reject invalid supervisor peer names");
11432 assert_eq!(cause, BridgeRejectionCause::InvalidSupervisorSpec);
11433 assert!(
11434 error.contains("invalid supervisor peer spec"),
11435 "bind rejection should explain invalid supervisor identity, got: {error}"
11436 );
11437 }
11438
11439 fn sample_bind_payload() -> meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
11440 meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
11441 supervisor: supervisor_bridge_spec(),
11442 epoch: 1,
11443 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
11444 expected_peer_id: PEER_ID_RECEIVER.to_string(),
11445 expected_address: "inproc://receiver".to_string(),
11446 bootstrap_token: "expected-token".into(),
11447 }
11448 }
11449
11450 async fn bind_material_adapter() -> (Arc<MeerkatMachine>, SessionId) {
11455 let adapter = Arc::new(MeerkatMachine::ephemeral());
11456 let session_id = SessionId::new();
11457 adapter
11458 .register_session(session_id.clone())
11459 .await
11460 .expect("register session");
11461 (adapter, session_id)
11462 }
11463
11464 #[tokio::test]
11470 async fn machine_owns_material_bind_admission_verdict_and_precedence() {
11471 use crate::meerkat_machine::dsl::SupervisorBindMaterialAdmissionKind as Verdict;
11472
11473 let (adapter, session_id) = bind_material_adapter().await;
11474
11475 let verdict = |address, sender, peer_id, token| {
11476 let adapter = adapter.clone();
11477 let session_id = session_id.clone();
11478 async move {
11479 adapter
11480 .resolve_supervisor_bind_material_admission(
11481 &session_id,
11482 address,
11483 sender,
11484 peer_id,
11485 token,
11486 )
11487 .await
11488 .expect("machine must emit a material bind-admission verdict")
11489 }
11490 };
11491
11492 assert_eq!(verdict(true, true, true, true).await, Verdict::Accept);
11494
11495 assert_eq!(
11497 verdict(false, true, true, true).await,
11498 Verdict::AddressMismatch
11499 );
11500 assert_eq!(
11501 verdict(true, false, true, true).await,
11502 Verdict::SenderMismatch
11503 );
11504 assert_eq!(
11505 verdict(true, true, false, true).await,
11506 Verdict::InvalidPeerSpec
11507 );
11508 assert_eq!(
11509 verdict(true, true, true, false).await,
11510 Verdict::InvalidBootstrapToken
11511 );
11512
11513 assert_eq!(
11515 verdict(false, false, false, false).await,
11516 Verdict::AddressMismatch
11517 );
11518 assert_eq!(
11520 verdict(true, false, false, false).await,
11521 Verdict::SenderMismatch
11522 );
11523 assert_eq!(
11525 verdict(true, true, false, false).await,
11526 Verdict::InvalidPeerSpec
11527 );
11528 }
11529
11530 #[tokio::test]
11531 async fn bridge_request_without_complete_peer_authority_fails_before_dispatch() {
11532 for install_peer_handle in [false, true] {
11533 let payload = sample_bind_payload();
11534 let sender = payload.supervisor.peer_id.clone();
11535 let mut runtime_impl = bootstrap_runtime(
11536 PEER_ID_RECEIVER,
11537 "inproc://receiver",
11538 Some("expected-token"),
11539 );
11540 let peer_handle = Arc::new(CountingPeerInteractionHandle::default());
11541 if install_peer_handle {
11542 let peer_handle: Arc<dyn meerkat_core::handles::PeerInteractionHandle> =
11543 peer_handle.clone();
11544 runtime_impl.peer_handle = Some(peer_handle);
11545 } else {
11546 runtime_impl.peer_handle = None;
11547 }
11548 runtime_impl.peer_request_response_handle = None;
11549 let completed_count = runtime_impl.completed_count.clone();
11550 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
11551 let adapter = Arc::new(MeerkatMachine::ephemeral());
11552 let session_id = SessionId::new();
11553 adapter
11554 .register_session(session_id.clone())
11555 .await
11556 .expect("register session");
11557 let candidate = bridge_candidate(&sender, &BridgeCommand::BindMember(payload));
11558
11559 assert!(
11560 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate)
11561 .await,
11562 "bridge handler must own bridge requests even when rejecting authority"
11563 );
11564 assert!(
11565 matches!(
11566 adapter.supervisor_binding(&session_id).await,
11567 SupervisorBinding::Unbound
11568 ),
11569 "bridge request must not mutate supervisor binding without complete authority"
11570 );
11571 assert_eq!(
11572 peer_handle.request_received_count(),
11573 0,
11574 "peer-only authority must not record PeerRequestReceived through bridge dispatch"
11575 );
11576 assert_eq!(
11577 completed_count.load(std::sync::atomic::Ordering::SeqCst),
11578 1,
11579 "bridge request rejected at the authority boundary should be marked complete"
11580 );
11581 }
11582 }
11583
11584 #[tokio::test]
11585 async fn bridge_request_rejected_by_peer_authority_fails_before_dispatch() {
11586 let payload = sample_bind_payload();
11587 let sender = payload.supervisor.peer_id.clone();
11588 let mut runtime_impl = bootstrap_runtime(
11589 PEER_ID_RECEIVER,
11590 "inproc://receiver",
11591 Some("expected-token"),
11592 );
11593 let peer_handle = Arc::new(CountingPeerInteractionHandle::rejecting_request_received());
11594 let peer_handle: Arc<dyn meerkat_core::handles::PeerInteractionHandle> = peer_handle;
11595 runtime_impl.peer_handle = Some(peer_handle.clone());
11596 runtime_impl.peer_request_response_handle = Some(peer_handle);
11597 let completed_count = runtime_impl.completed_count.clone();
11598 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
11599 let adapter = Arc::new(MeerkatMachine::ephemeral());
11600 let session_id = SessionId::new();
11601 adapter
11602 .register_session(session_id.clone())
11603 .await
11604 .expect("register session");
11605 let candidate = bridge_candidate(&sender, &BridgeCommand::BindMember(payload));
11606
11607 assert!(
11608 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate).await,
11609 "bridge handler must own bridge requests even when rejecting authority"
11610 );
11611 assert!(
11612 matches!(
11613 adapter.supervisor_binding(&session_id).await,
11614 SupervisorBinding::Unbound
11615 ),
11616 "bridge request must not mutate supervisor binding when PeerRequestReceived is rejected"
11617 );
11618 assert_eq!(
11619 completed_count.load(std::sync::atomic::Ordering::SeqCst),
11620 1,
11621 "bridge request rejected by peer authority should be marked complete"
11622 );
11623 }
11624
11625 async fn bind_sample_supervisor(
11626 adapter: &MeerkatMachine,
11627 session_id: &SessionId,
11628 payload: &meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload,
11629 ) {
11630 let spec = TrustedPeerDescriptor::try_from(payload.supervisor.clone())
11631 .expect("valid supervisor spec");
11632 adapter
11633 .stage_supervisor_bind(
11634 session_id,
11635 spec.name.to_string(),
11636 spec.peer_id.as_str(),
11637 spec.address.to_string(),
11638 signing_public_key_for_descriptor(&spec),
11639 payload.epoch,
11640 )
11641 .await
11642 .expect("supervisor bound");
11643 }
11644
11645 async fn stage_test_supervisor_rotation(
11646 adapter: &MeerkatMachine,
11647 session_id: &SessionId,
11648 next: crate::meerkat_machine::GeneratedSupervisorBinding,
11649 ) {
11650 let SupervisorBinding::Bound {
11651 peer_id: previous_peer_id,
11652 signing_public_key: previous_signing_public_key,
11653 epoch: previous_epoch,
11654 ..
11655 } = adapter.supervisor_binding(session_id).await
11656 else {
11657 panic!("test rotation requires a bound previous supervisor");
11658 };
11659 let operation_id = uuid::Uuid::new_v4().to_string();
11660 let submission = adapter
11661 .submit_supervisor_rotation(
11662 session_id,
11663 GeneratedSupervisorRotationSubmit {
11664 operation_id: operation_id.clone(),
11665 next: next.clone(),
11666 preflight_rejection: None,
11667 sender_peer_id: Some(previous_peer_id.clone()),
11668 sender_signing_public_key: Some(previous_signing_public_key),
11669 },
11670 )
11671 .await
11672 .expect("stage test rotation submit");
11673 assert!(matches!(submission, SupervisorRotationSubmission::New(_)));
11674 adapter
11675 .stage_supervisor_rotation_previous_revoked(
11676 session_id,
11677 operation_id.clone(),
11678 previous_peer_id,
11679 previous_epoch,
11680 )
11681 .await
11682 .expect("stage test rotation previous-revoked checkpoint");
11683 adapter
11684 .stage_supervisor_rotation_next_published(
11685 session_id,
11686 operation_id,
11687 next.peer_id,
11688 next.epoch,
11689 )
11690 .await
11691 .expect("stage test rotation bound checkpoint");
11692 }
11693
11694 #[tokio::test]
11695 async fn generated_bind_admission_allows_bootstrap_when_unbound() {
11696 let payload = sample_bind_payload();
11697 let supervisor = bridge_peer_identity(&payload.supervisor, "test").expect("valid peer");
11698 let adapter = MeerkatMachine::ephemeral();
11699 let session_id = SessionId::new();
11700 adapter
11701 .register_session(session_id.clone())
11702 .await
11703 .expect("register session");
11704 let sender = bridge_sender_fact(&payload.supervisor.peer_id);
11705 let admission = adapter
11706 .resolve_supervisor_bind_admission(
11707 &session_id,
11708 supervisor.peer_id.as_str().clone(),
11709 payload.epoch,
11710 generated_sender_peer_id(&sender),
11711 )
11712 .await
11713 .expect("generated bind admission");
11714 assert!(matches!(admission, SupervisorBindAdmission::Bootstrap));
11715 }
11716
11717 #[tokio::test]
11718 async fn generated_bind_admission_rejects_different_supervisor_takeover() {
11719 let current_payload = sample_bind_payload();
11723 let adapter = MeerkatMachine::ephemeral();
11724 let session_id = SessionId::new();
11725 adapter
11726 .register_session(session_id.clone())
11727 .await
11728 .expect("register session");
11729 bind_sample_supervisor(&adapter, &session_id, ¤t_payload).await;
11730 let mut takeover = sample_bind_payload();
11731 takeover.supervisor = old_supervisor_bridge_spec();
11732 let supervisor = bridge_peer_identity(&takeover.supervisor, "test").expect("valid peer");
11733 let sender = bridge_sender_fact(&takeover.supervisor.peer_id);
11734 let admission = adapter
11735 .resolve_supervisor_bind_admission(
11736 &session_id,
11737 supervisor.peer_id.as_str().clone(),
11738 takeover.epoch,
11739 generated_sender_peer_id(&sender),
11740 )
11741 .await
11742 .expect("generated bind admission");
11743 assert!(
11744 matches!(
11745 admission,
11746 SupervisorBindAdmission::Rejected(
11747 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::AlreadyBound
11748 )
11749 ),
11750 "different-supervisor bind must reject as already bound: {admission:?}"
11751 );
11752 }
11753
11754 #[tokio::test]
11755 async fn generated_bind_admission_rejects_lower_epoch_replay() {
11756 let current_payload = sample_bind_payload();
11760 let adapter = MeerkatMachine::ephemeral();
11761 let session_id = SessionId::new();
11762 adapter
11763 .register_session(session_id.clone())
11764 .await
11765 .expect("register session");
11766 bind_sample_supervisor(&adapter, &session_id, ¤t_payload).await;
11767 let mut replay = sample_bind_payload();
11768 replay.epoch = current_payload.epoch - 1;
11769 let supervisor = bridge_peer_identity(&replay.supervisor, "test").expect("valid peer");
11770 let sender = bridge_sender_fact(&replay.supervisor.peer_id);
11771 let admission = adapter
11772 .resolve_supervisor_bind_admission(
11773 &session_id,
11774 supervisor.peer_id.as_str().clone(),
11775 replay.epoch,
11776 generated_sender_peer_id(&sender),
11777 )
11778 .await
11779 .expect("generated bind admission");
11780 assert!(
11781 matches!(
11782 admission,
11783 SupervisorBindAdmission::Rejected(
11784 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::AlreadyBound
11785 )
11786 ),
11787 "lower-epoch bind replay must reject as already bound: {admission:?}"
11788 );
11789 }
11790
11791 #[tokio::test]
11792 async fn generated_bind_admission_rejects_higher_epoch_same_supervisor_rebind() {
11793 let current_payload = sample_bind_payload();
11797 let adapter = MeerkatMachine::ephemeral();
11798 let session_id = SessionId::new();
11799 adapter
11800 .register_session(session_id.clone())
11801 .await
11802 .expect("register session");
11803 bind_sample_supervisor(&adapter, &session_id, ¤t_payload).await;
11804 let mut advance = sample_bind_payload();
11805 advance.epoch = current_payload.epoch + 5;
11806 let supervisor = bridge_peer_identity(&advance.supervisor, "test").expect("valid peer");
11807 let sender = bridge_sender_fact(&advance.supervisor.peer_id);
11808 let admission = adapter
11809 .resolve_supervisor_bind_admission(
11810 &session_id,
11811 supervisor.peer_id.as_str().clone(),
11812 advance.epoch,
11813 generated_sender_peer_id(&sender),
11814 )
11815 .await
11816 .expect("generated bind admission");
11817 assert!(
11818 matches!(
11819 admission,
11820 SupervisorBindAdmission::Rejected(
11821 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::AlreadyBound
11822 )
11823 ),
11824 "higher-epoch rebind must reject as already bound: {admission:?}"
11825 );
11826 }
11827
11828 #[tokio::test]
11829 async fn generated_bind_admission_rejects_spoofed_sender() {
11830 let current_payload = sample_bind_payload();
11834 let adapter = MeerkatMachine::ephemeral();
11835 let session_id = SessionId::new();
11836 adapter
11837 .register_session(session_id.clone())
11838 .await
11839 .expect("register session");
11840 bind_sample_supervisor(&adapter, &session_id, ¤t_payload).await;
11841 let retry = sample_bind_payload();
11842 let supervisor = bridge_peer_identity(&retry.supervisor, "test").expect("valid peer");
11843 let attacker_peer_id = PeerId::new().as_str();
11844 let sender = bridge_sender_fact(&attacker_peer_id);
11845 let admission = adapter
11846 .resolve_supervisor_bind_admission(
11847 &session_id,
11848 supervisor.peer_id.as_str().clone(),
11849 retry.epoch,
11850 generated_sender_peer_id(&sender),
11851 )
11852 .await
11853 .expect("generated bind admission");
11854 assert!(
11855 matches!(
11856 admission,
11857 SupervisorBindAdmission::Rejected(
11858 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::SenderMismatch
11859 )
11860 ),
11861 "bind from unauthorized sender must reject as sender mismatch: {admission:?}"
11862 );
11863 }
11864
11865 #[tokio::test]
11866 async fn generated_bind_admission_rejects_same_display_name_different_canonical_sender() {
11867 let current_payload = sample_bind_payload();
11868 let adapter = MeerkatMachine::ephemeral();
11869 let session_id = SessionId::new();
11870 adapter
11871 .register_session(session_id.clone())
11872 .await
11873 .expect("register session");
11874 bind_sample_supervisor(&adapter, &session_id, ¤t_payload).await;
11875 let retry = sample_bind_payload();
11876 let supervisor = bridge_peer_identity(&retry.supervisor, "test").expect("valid peer");
11877 let attacker_peer_id =
11878 PeerId::parse(PEER_ID_OLD_SUPERVISOR).expect("valid attacker peer id");
11879 let sender = bridge_sender_fact_with_display(attacker_peer_id, &retry.supervisor.name);
11880
11881 let admission = adapter
11882 .resolve_supervisor_bind_admission(
11883 &session_id,
11884 supervisor.peer_id.as_str().clone(),
11885 retry.epoch,
11886 generated_sender_peer_id(&sender),
11887 )
11888 .await
11889 .expect("generated bind admission");
11890 assert!(
11891 matches!(
11892 admission,
11893 SupervisorBindAdmission::Rejected(
11894 crate::meerkat_machine::dsl::SupervisorBindRejectionKind::SenderMismatch
11895 )
11896 ),
11897 "same display name with different canonical sender must reject: {admission:?}"
11898 );
11899 }
11900
11901 #[tokio::test]
11902 async fn generated_bind_admission_idempotently_acks_retry_from_current_supervisor() {
11903 let current_payload = sample_bind_payload();
11907 let adapter = MeerkatMachine::ephemeral();
11908 let session_id = SessionId::new();
11909 adapter
11910 .register_session(session_id.clone())
11911 .await
11912 .expect("register session");
11913 bind_sample_supervisor(&adapter, &session_id, ¤t_payload).await;
11914 let retry = sample_bind_payload();
11915 let supervisor = bridge_peer_identity(&retry.supervisor, "test").expect("valid peer");
11916 let sender = bridge_sender_fact(&retry.supervisor.peer_id);
11917 let admission = adapter
11918 .resolve_supervisor_bind_admission(
11919 &session_id,
11920 supervisor.peer_id.as_str().clone(),
11921 retry.epoch,
11922 generated_sender_peer_id(&sender),
11923 )
11924 .await
11925 .expect("generated bind admission");
11926 assert!(matches!(admission, SupervisorBindAdmission::IdempotentAck));
11927 }
11928
11929 #[tokio::test]
11930 async fn bind_member_handler_rejects_rebind_after_supervisor_bound() {
11931 let runtime: Arc<dyn CommsRuntime> = Arc::new(
11935 meerkat_comms::CommsRuntime::inproc_only("bind-rebind-receiver")
11936 .expect("receiver runtime"),
11937 );
11938 let supervisor_runtime = Arc::new(
11939 meerkat_comms::CommsRuntime::inproc_only("mob/__mob_supervisor__")
11940 .expect("supervisor runtime"),
11941 );
11942 let adapter = Arc::new(MeerkatMachine::ephemeral());
11943 let session_id = SessionId::new();
11944 adapter
11945 .register_session(session_id.clone())
11946 .await
11947 .expect("register session");
11948 let current_supervisor =
11949 trusted_peer_from_runtime("mob/__mob_supervisor__", &supervisor_runtime);
11950 adapter
11951 .stage_supervisor_bind(
11952 &session_id,
11953 current_supervisor.name.to_string(),
11954 current_supervisor.peer_id.as_str(),
11955 current_supervisor.address.to_string(),
11956 signing_public_key_for_descriptor(¤t_supervisor),
11957 1,
11958 )
11959 .await
11960 .expect("initial bind must succeed");
11961 let adversary_supervisor = old_supervisor_bridge_spec();
11962 let adversary_peer_id = adversary_supervisor.peer_id.clone();
11963 let command = BridgeCommand::BindMember(
11964 meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
11965 supervisor: adversary_supervisor,
11966 epoch: 2,
11967 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
11968 expected_peer_id: runtime
11969 .peer_id()
11970 .map(|peer_id| peer_id.as_str())
11971 .unwrap_or_else(|| PEER_ID_RECEIVER.to_string()),
11972 expected_address: runtime
11973 .advertised_address()
11974 .unwrap_or_else(|| "inproc://bind-rebind-receiver".to_string()),
11975 bootstrap_token: runtime.bridge_bootstrap_token().unwrap_or_default().into(),
11977 },
11978 );
11979 let candidate = bridge_candidate(&adversary_peer_id, &command);
11980
11981 assert!(
11982 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate,)
11983 .await,
11984 "bridge handler must own the BindMember command"
11985 );
11986 let binding = adapter.supervisor_binding(&session_id).await;
11987 let SupervisorBinding::Bound { peer_id, epoch, .. } = binding else {
11988 panic!("supervisor binding must be preserved as Bound");
11989 };
11990 assert_eq!(
11991 peer_id,
11992 current_supervisor.peer_id.as_str(),
11993 "rebind attempt must not replace the authorized supervisor"
11994 );
11995 assert_eq!(
11996 epoch, 1,
11997 "rebind attempt must not advance the authorized epoch"
11998 );
11999 }
12000
12001 #[tokio::test]
12010 async fn bind_member_handler_rebind_reply_is_typed_already_bound() {
12011 let sent: Arc<tokio::sync::Mutex<Vec<CommsCommand>>> =
12012 Arc::new(tokio::sync::Mutex::new(Vec::new()));
12013 let runtime: Arc<dyn CommsRuntime> = Arc::new(CapturingRuntime {
12014 peer_id: PEER_ID_RECEIVER.to_string(),
12015 advertised_address: Some("inproc://receiver".to_string()),
12016 bootstrap_token: Some("expected-token".to_string()),
12017 inbox_notify: Arc::new(tokio::sync::Notify::new()),
12018 sent: sent.clone(),
12019 });
12020 let adapter = Arc::new(MeerkatMachine::ephemeral());
12021 let session_id = SessionId::new();
12022 adapter
12023 .register_session(session_id.clone())
12024 .await
12025 .expect("register session");
12026 let current = trusted_supervisor_descriptor(0xbb);
12027 adapter
12028 .stage_supervisor_bind(
12029 &session_id,
12030 current.name.to_string(),
12031 current.peer_id.as_str(),
12032 current.address.to_string(),
12033 signing_public_key_for_descriptor(¤t),
12034 1,
12035 )
12036 .await
12037 .expect("initial bind must succeed");
12038
12039 let adversary = old_supervisor_bridge_spec();
12043 let adversary_peer_id = adversary.peer_id.clone();
12044 let command = BridgeCommand::BindMember(
12045 meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
12046 supervisor: adversary,
12047 epoch: 2,
12048 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
12049 expected_peer_id: PEER_ID_RECEIVER.to_string(),
12050 expected_address: "inproc://receiver".to_string(),
12051 bootstrap_token: "expected-token".into(),
12052 },
12053 );
12054 let candidate = bridge_candidate(&adversary_peer_id, &command);
12055
12056 assert!(
12057 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate,)
12058 .await,
12059 "bridge handler must own the BindMember command"
12060 );
12061
12062 let binding = adapter.supervisor_binding(&session_id).await;
12064 let SupervisorBinding::Bound { peer_id, .. } = binding else {
12065 panic!("binding preserved");
12066 };
12067 assert_eq!(peer_id, current.peer_id.as_str());
12068
12069 let (result, status) = sent
12071 .lock()
12072 .await
12073 .iter()
12074 .find_map(|cmd| match cmd {
12075 CommsCommand::PeerResponse { result, status, .. } => {
12076 Some((result.clone(), *status))
12077 }
12078 _ => None,
12079 })
12080 .expect("handler must send a PeerResponse for the rejection");
12081 assert!(
12082 matches!(status, meerkat_core::interaction::ResponseStatus::Failed),
12083 "rebind rejection must surface as Failed status"
12084 );
12085 let reply: BridgeReply = serde_json::from_value(result).expect("typed bridge reply");
12086 match reply {
12087 BridgeReply::Rejected { cause, .. } => {
12088 assert_eq!(
12089 cause,
12090 BridgeRejectionCause::AlreadyBound,
12091 "different-supervisor rebind must be rejected as AlreadyBound",
12092 );
12093 }
12094 other => unreachable!("expected Rejected reply, got {other:?}"),
12095 }
12096 }
12097
12098 struct CapturingRuntime {
12103 peer_id: String,
12104 advertised_address: Option<String>,
12105 bootstrap_token: Option<String>,
12106 inbox_notify: Arc<tokio::sync::Notify>,
12107 sent: Arc<tokio::sync::Mutex<Vec<CommsCommand>>>,
12108 }
12109
12110 #[async_trait::async_trait]
12111 impl CommsRuntime for CapturingRuntime {
12112 fn peer_id(&self) -> Option<PeerId> {
12113 PeerId::parse(&self.peer_id).ok()
12114 }
12115
12116 fn public_key(&self) -> Option<String> {
12117 test_public_key_for_peer_id(&self.peer_id)
12118 }
12119
12120 fn public_key_bytes(&self) -> Option<[u8; 32]> {
12121 test_pubkey_bytes_for_peer_id(&self.peer_id)
12122 }
12123
12124 fn comms_name(&self) -> Option<String> {
12125 self.advertised_address
12126 .as_deref()
12127 .map(test_comms_name_from_address)
12128 }
12129
12130 fn advertised_address(&self) -> Option<String> {
12131 self.advertised_address.clone()
12132 }
12133
12134 fn bridge_bootstrap_token(&self) -> Option<String> {
12135 self.bootstrap_token.clone()
12136 }
12137
12138 async fn drain_messages(&self) -> Vec<String> {
12139 Vec::new()
12140 }
12141
12142 fn inbox_notify(&self) -> Arc<tokio::sync::Notify> {
12143 self.inbox_notify.clone()
12144 }
12145
12146 fn peer_request_response_authority_handle(
12147 &self,
12148 ) -> Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>> {
12149 Some(Arc::new(CountingPeerInteractionHandle::default()))
12150 }
12151
12152 async fn apply_trust_mutation(
12153 &self,
12154 mutation: CommsTrustMutation,
12155 ) -> Result<CommsTrustMutationResult, SendError> {
12156 match mutation {
12157 CommsTrustMutation::AddPrivateTrustedPeer { peer, authority } => {
12158 authority
12159 .validate_private_add(self.peer_id(), &peer)
12160 .map_err(SendError::Validation)?;
12161 Ok(CommsTrustMutationResult::Added { created: true })
12162 }
12163 CommsTrustMutation::RemovePrivateTrustedPeer { peer_id, authority } => {
12164 let parsed_peer_id = PeerId::parse(&peer_id)
12165 .map_err(|error| SendError::Validation(error.to_string()))?;
12166 authority
12167 .validate_private_remove(self.peer_id(), parsed_peer_id)
12168 .map_err(SendError::Validation)?;
12169 Ok(CommsTrustMutationResult::Removed { removed: true })
12170 }
12171 _ => Err(SendError::Unsupported(
12172 "test runtime only supports generated private trust".into(),
12173 )),
12174 }
12175 }
12176
12177 async fn send(
12178 &self,
12179 cmd: CommsCommand,
12180 ) -> Result<meerkat_core::comms::SendReceipt, SendError> {
12181 let receipt = match &cmd {
12182 CommsCommand::PeerResponse { in_reply_to, .. } => {
12183 meerkat_core::comms::SendReceipt::PeerResponseSent {
12184 envelope_id: Uuid::new_v4(),
12185 in_reply_to: *in_reply_to,
12186 }
12187 }
12188 _ => meerkat_core::comms::SendReceipt::PeerMessageSent {
12189 envelope_id: Uuid::new_v4(),
12190 delivery: meerkat_core::comms::PeerDeliveryOutcome::Acked,
12191 },
12192 };
12193 self.sent.lock().await.push(cmd);
12194 Ok(receipt)
12195 }
12196 }
12197
12198 #[tokio::test]
12199 async fn bind_member_handler_response_reports_canonical_protocol_versions() {
12200 let sent: Arc<tokio::sync::Mutex<Vec<CommsCommand>>> =
12201 Arc::new(tokio::sync::Mutex::new(Vec::new()));
12202 let runtime: Arc<dyn CommsRuntime> = Arc::new(CapturingRuntime {
12203 peer_id: PEER_ID_RECEIVER.to_string(),
12204 advertised_address: Some("inproc://receiver".to_string()),
12205 bootstrap_token: Some("expected-token".to_string()),
12206 inbox_notify: Arc::new(tokio::sync::Notify::new()),
12207 sent: sent.clone(),
12208 });
12209 let adapter = Arc::new(MeerkatMachine::ephemeral());
12210 let session_id = SessionId::new();
12211 adapter
12212 .register_session(session_id.clone())
12213 .await
12214 .expect("register session");
12215 let supervisor = supervisor_bridge_spec();
12216 let command = BridgeCommand::BindMember(
12217 meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
12218 supervisor: supervisor.clone(),
12219 epoch: 0,
12220 protocol_version: supervisor_bridge_current_protocol_version(),
12221 expected_peer_id: PEER_ID_RECEIVER.to_string(),
12222 expected_address: "inproc://receiver".to_string(),
12223 bootstrap_token: "expected-token".into(),
12224 },
12225 );
12226 let candidate = bridge_candidate(&supervisor.peer_id, &command);
12227
12228 assert!(
12229 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate,)
12230 .await,
12231 "bridge handler must own the BindMember command"
12232 );
12233
12234 let (result, status) = sent
12235 .lock()
12236 .await
12237 .iter()
12238 .find_map(|cmd| match cmd {
12239 CommsCommand::PeerResponse { result, status, .. } => {
12240 Some((result.clone(), *status))
12241 }
12242 _ => None,
12243 })
12244 .expect("handler must send a bind response");
12245 assert!(matches!(
12246 status,
12247 meerkat_core::interaction::ResponseStatus::Completed
12248 ));
12249 let reply: BridgeReply = serde_json::from_value(result).expect("typed bridge reply");
12250 let BridgeReply::BindMember(response) = reply else {
12251 panic!("expected bind response");
12252 };
12253 assert_eq!(
12254 response.capabilities.current_protocol_version,
12255 supervisor_bridge_current_protocol_version()
12256 );
12257 assert_eq!(
12258 response.capabilities.default_protocol_version,
12259 supervisor_bridge_default_protocol_version()
12260 );
12261 assert_eq!(
12262 response.capabilities.supported_protocol_versions,
12263 supervisor_bridge_supported_protocol_versions()
12264 );
12265 assert!(
12266 response.capabilities.hard_cancel_member,
12267 "bind response advertises the phase-6 hard-cancel arm"
12268 );
12269 assert!(
12270 response.capabilities.tracked_input_cancel,
12271 "bind response advertises exact durable tracked-input cancellation"
12272 );
12273 }
12274
12275 #[tokio::test]
12276 async fn idempotent_ack_invariant_rejects_without_echoing_attacker_fields() {
12277 let sent: Arc<tokio::sync::Mutex<Vec<CommsCommand>>> =
12284 Arc::new(tokio::sync::Mutex::new(Vec::new()));
12285 let runtime: Arc<dyn CommsRuntime> = Arc::new(CapturingRuntime {
12286 peer_id: PEER_ID_RECEIVER.to_string(),
12287 advertised_address: None,
12290 bootstrap_token: Some("expected-token".to_string()),
12291 inbox_notify: Arc::new(tokio::sync::Notify::new()),
12292 sent: sent.clone(),
12293 });
12294 let adapter = Arc::new(MeerkatMachine::ephemeral());
12295 let session_id = SessionId::new();
12296 adapter
12297 .register_session(session_id.clone())
12298 .await
12299 .expect("register session");
12300 let authorized = trusted_supervisor_descriptor(0xbb);
12301 adapter
12302 .stage_supervisor_bind(
12303 &session_id,
12304 authorized.name.to_string(),
12305 authorized.peer_id.as_str(),
12306 authorized.address.to_string(),
12307 signing_public_key_for_descriptor(&authorized),
12308 7,
12309 )
12310 .await
12311 .expect("pre-bind supervisor");
12312 let attacker_address = "inproc://ATTACKER-ADDRESS-DO-NOT-ECHO".to_string();
12315 let attacker_peer_id = format!("{}-ATTACKER-PEER-ID-DO-NOT-ECHO", PeerId::new().as_str());
12316 let command = BridgeCommand::BindMember(
12317 meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
12318 supervisor: BridgePeerSpec::from(authorized.clone()),
12322 epoch: 7,
12323 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
12324 expected_peer_id: attacker_peer_id.clone(),
12325 expected_address: attacker_address.clone(),
12326 bootstrap_token: "expected-token".into(),
12327 },
12328 );
12329 let candidate = bridge_candidate(&authorized.peer_id.as_str(), &command);
12330
12331 assert!(
12332 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate,)
12333 .await,
12334 "bridge handler must own the BindMember command"
12335 );
12336
12337 let binding = adapter.supervisor_binding(&session_id).await;
12340 let SupervisorBinding::Bound {
12341 peer_id: stored_peer_id,
12342 epoch: stored_epoch,
12343 ..
12344 } = binding
12345 else {
12346 panic!("binding must survive invariant failure");
12347 };
12348 assert_eq!(stored_peer_id, authorized.peer_id.as_str());
12349 assert_eq!(stored_epoch, 7);
12350
12351 let sent_commands = sent.lock().await.clone();
12354 let (result, status) = sent_commands
12355 .into_iter()
12356 .find_map(|cmd| match cmd {
12357 CommsCommand::PeerResponse { result, status, .. } => Some((result, status)),
12358 _ => None,
12359 })
12360 .expect("handler must send a PeerResponse for the invariant violation");
12361 assert!(
12362 matches!(status, meerkat_core::interaction::ResponseStatus::Failed),
12363 "invariant violation must surface as Failed status"
12364 );
12365 let reply: BridgeReply =
12366 serde_json::from_value(result.clone()).expect("typed bridge reply");
12367 assert!(
12368 matches!(reply, BridgeReply::Rejected { .. }),
12369 "expected Rejected reply for invariant violation, got: {reply:?}"
12370 );
12371 if let BridgeReply::Rejected { cause, reason } = reply {
12372 assert_eq!(cause, BridgeRejectionCause::Internal);
12373 assert!(
12374 !reason.contains(&attacker_address),
12375 "rejection reason must not echo attacker-supplied address: {reason}"
12376 );
12377 assert!(
12378 !reason.contains(&attacker_peer_id),
12379 "rejection reason must not echo attacker-supplied peer_id: {reason}"
12380 );
12381 }
12382 let raw = serde_json::to_string(&result).expect("serialize reply for attacker-field check");
12383 assert!(
12384 !raw.contains(&attacker_address),
12385 "reply payload must not contain attacker-supplied address: {raw}"
12386 );
12387 assert!(
12388 !raw.contains(&attacker_peer_id),
12389 "reply payload must not contain attacker-supplied peer_id: {raw}"
12390 );
12391 }
12392
12393 #[tokio::test]
12394 async fn generated_authorize_admission_rejects_initial_claim_without_bind() {
12395 let payload = BridgeSupervisorPayload {
12396 supervisor: supervisor_bridge_spec(),
12397 epoch: 0,
12398 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
12399 };
12400 let supervisor = bridge_peer_identity(&payload.supervisor, "test").expect("valid peer");
12401 let adapter = MeerkatMachine::ephemeral();
12402 let session_id = SessionId::new();
12403 adapter
12404 .register_session(session_id.clone())
12405 .await
12406 .expect("register session");
12407 let sender = bridge_sender_fact(&payload.supervisor.peer_id);
12408
12409 assert!(
12410 matches!(
12411 adapter
12412 .resolve_supervisor_authorize_admission(
12413 &session_id,
12414 GeneratedSupervisorBinding {
12415 name: supervisor.name.to_string(),
12416 peer_id: supervisor.peer_id.as_str().clone(),
12417 address: supervisor.address.to_string(),
12418 signing_public_key: encode_supervisor_signing_public_key(
12419 *supervisor.pubkey.as_bytes(),
12420 ),
12421 epoch: payload.epoch,
12422 },
12423 generated_sender_peer_id(&sender),
12424 generated_sender_signing_public_key(&sender),
12425 )
12426 .await
12427 .expect("generated authorize admission"),
12428 SupervisorAuthorizeAdmission::Rejected(
12429 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::NotBound
12430 )
12431 ),
12432 "first supervisor claim must go through bind_member"
12433 );
12434 }
12435
12436 #[tokio::test]
12437 async fn generated_authorize_admission_rejects_same_display_name_different_canonical_sender() {
12438 let current_payload = sample_bind_payload();
12439 let payload = BridgeSupervisorPayload {
12440 supervisor: current_supervisor_bridge_spec(),
12441 epoch: current_payload.epoch + 1,
12442 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
12443 };
12444 let supervisor = bridge_peer_identity(&payload.supervisor, "test").expect("valid peer");
12445 let adapter = MeerkatMachine::ephemeral();
12446 let session_id = SessionId::new();
12447 adapter
12448 .register_session(session_id.clone())
12449 .await
12450 .expect("register session");
12451 bind_sample_supervisor(&adapter, &session_id, ¤t_payload).await;
12452 let attacker_peer_id =
12453 PeerId::parse(PEER_ID_OLD_SUPERVISOR).expect("valid attacker peer id");
12454 let sender =
12455 bridge_sender_fact_with_display(attacker_peer_id, ¤t_payload.supervisor.name);
12456
12457 assert!(
12458 matches!(
12459 adapter
12460 .resolve_supervisor_authorize_admission(
12461 &session_id,
12462 GeneratedSupervisorBinding {
12463 name: supervisor.name.to_string(),
12464 peer_id: supervisor.peer_id.as_str().clone(),
12465 address: supervisor.address.to_string(),
12466 signing_public_key: encode_supervisor_signing_public_key(
12467 *supervisor.pubkey.as_bytes(),
12468 ),
12469 epoch: payload.epoch,
12470 },
12471 generated_sender_peer_id(&sender),
12472 generated_sender_signing_public_key(&sender),
12473 )
12474 .await
12475 .expect("generated authorize admission"),
12476 SupervisorAuthorizeAdmission::Rejected(
12477 crate::meerkat_machine::dsl::SupervisorAuthorizeRejectionKind::SenderMismatch
12478 )
12479 ),
12480 "same display name with a different canonical sender must reject"
12481 );
12482 }
12483
12484 #[tokio::test]
12485 async fn supervisor_bridge_admission_rejects_same_display_name_different_canonical_sender() {
12486 let current_payload = sample_bind_payload();
12487 let payload = BridgeSupervisorPayload {
12488 supervisor: current_payload.supervisor.clone(),
12489 epoch: current_payload.epoch,
12490 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
12491 };
12492 let attacker_peer_id =
12493 PeerId::parse(PEER_ID_OLD_SUPERVISOR).expect("valid attacker peer id");
12494 let sender =
12495 bridge_sender_fact_with_display(attacker_peer_id, ¤t_payload.supervisor.name);
12496
12497 let adapter = MeerkatMachine::ephemeral();
12498 let session_id = SessionId::new();
12499 adapter
12500 .register_session(session_id.clone())
12501 .await
12502 .expect("register session");
12503 let supervisor = TrustedPeerDescriptor::try_from(current_payload.supervisor.clone())
12504 .expect("valid supervisor spec");
12505 adapter
12506 .stage_supervisor_bind(
12507 &session_id,
12508 supervisor.name.to_string(),
12509 supervisor.peer_id.as_str(),
12510 supervisor.address.to_string(),
12511 signing_public_key_for_descriptor(&supervisor),
12512 current_payload.epoch,
12513 )
12514 .await
12515 .expect("supervisor bound");
12516
12517 let (cause, error) =
12518 resolve_authorized_supervisor(&adapter, &session_id, &sender, &payload)
12519 .await
12520 .expect_err("same display name with a different canonical sender must reject");
12521 assert_eq!(cause, BridgeRejectionCause::SenderMismatch);
12522 assert!(
12523 error.contains("MeerkatMachine"),
12524 "supervisor command rejection should explain the sender mismatch, got: {error}"
12525 );
12526 }
12527
12528 #[tokio::test]
12538 async fn validate_bind_request_rejects_empty_bootstrap_token_at_runtime() {
12539 let runtime: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
12540 PEER_ID_RECEIVER,
12541 "inproc://receiver",
12542 Some(""),
12543 ));
12544 let (adapter, session_id) = bind_material_adapter().await;
12545 let supervisor = supervisor_bridge_spec();
12546 let payload = meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
12547 supervisor: supervisor.clone(),
12548 epoch: 0,
12549 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
12550 expected_peer_id: PEER_ID_RECEIVER.to_string(),
12551 expected_address: runtime.advertised_address().unwrap(),
12552 bootstrap_token: "whatever".into(),
12553 };
12554
12555 let (cause, _error) = validate_bind_request(
12556 &adapter,
12557 &session_id,
12558 &runtime,
12559 &bridge_sender_fact(&supervisor.peer_id),
12560 &payload,
12561 )
12562 .await
12563 .expect_err("runtime with empty token must refuse to validate");
12564 assert_eq!(cause, BridgeRejectionCause::InvalidBootstrapToken);
12565 }
12566
12567 #[tokio::test]
12568 async fn validate_bind_request_rejects_query_string_bootstrap_without_typed_runtime_token() {
12569 let runtime: Arc<dyn CommsRuntime> = Arc::new(bootstrap_runtime(
12570 PEER_ID_RECEIVER,
12571 &format!("inproc://receiver?{SUPERVISOR_BRIDGE_BOOTSTRAP_TOKEN_PARAM}=expected-token"),
12572 None,
12573 ));
12574 let (adapter, session_id) = bind_material_adapter().await;
12575 let supervisor = supervisor_bridge_spec();
12576 let payload = meerkat_contracts::wire::supervisor_bridge::BridgeBindPayload {
12577 supervisor: supervisor.clone(),
12578 epoch: 0,
12579 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
12580 expected_peer_id: PEER_ID_RECEIVER.to_string(),
12581 expected_address: "inproc://receiver".to_string(),
12582 bootstrap_token: "expected-token".into(),
12583 };
12584
12585 let (cause, error) = validate_bind_request(
12586 &adapter,
12587 &session_id,
12588 &runtime,
12589 &bridge_sender_fact(&supervisor.peer_id),
12590 &payload,
12591 )
12592 .await
12593 .expect_err("query-string token must not satisfy typed bootstrap proof");
12594 assert_eq!(cause, BridgeRejectionCause::InvalidBootstrapToken);
12595 assert!(
12596 error.contains("typed bridge bootstrap token"),
12597 "runtime should explain that the typed token field is required, got: {error}"
12598 );
12599 }
12600
12601 fn bridge_candidate(sender: &str, command: &BridgeCommand) -> PeerInputCandidate {
12602 let id = InteractionId(Uuid::new_v4());
12603 let ingress = bridge_sender_fact_with_id(id, sender);
12604 bridge_candidate_with_ingress(sender, command, ingress)
12605 }
12606
12607 fn bridge_candidate_with_ingress(
12608 sender: &str,
12609 command: &BridgeCommand,
12610 ingress: PeerIngressFact,
12611 ) -> PeerInputCandidate {
12612 let id = ingress.interaction_id;
12613 PeerInputCandidate {
12614 interaction: InboxInteraction {
12615 objective_id: None,
12616 sender_taint: None,
12617 id,
12618 from_route: None,
12619 from: sender.to_string(),
12620 content: InteractionContent::Request {
12621 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
12622 params: serde_json::to_value(command).expect("serialize bridge command"),
12623 blocks: None,
12624 },
12625 rendered_text: String::new(),
12626 handling_mode: HandlingMode::Queue,
12627 render_metadata: None,
12628 },
12629 ingress,
12630 lifecycle_peer: None,
12631 response_terminality: None,
12632 }
12633 }
12634
12635 fn bridge_delivery_candidate_with_ingress(
12636 sender: &str,
12637 delivery: &serde_json::Value,
12638 ingress: PeerIngressFact,
12639 ) -> PeerInputCandidate {
12640 bridge_delivery_raw_candidate_with_ingress(
12641 sender,
12642 serde_json::to_string(delivery).expect("serialize bridge delivery body"),
12643 ingress,
12644 )
12645 }
12646
12647 fn bridge_delivery_raw_candidate_with_ingress(
12648 sender: &str,
12649 body: String,
12650 ingress: PeerIngressFact,
12651 ) -> PeerInputCandidate {
12652 let id = ingress.interaction_id;
12653 PeerInputCandidate {
12654 interaction: InboxInteraction {
12655 objective_id: None,
12656 sender_taint: None,
12657 id,
12658 from_route: None,
12659 from: sender.to_string(),
12660 content: InteractionContent::Message { body, blocks: None },
12661 rendered_text: String::new(),
12662 handling_mode: HandlingMode::Queue,
12663 render_metadata: None,
12664 },
12665 ingress,
12666 lifecycle_peer: None,
12667 response_terminality: None,
12668 }
12669 }
12670
12671 #[tokio::test]
12672 async fn raw_rotation_marker_versions_reject_durably_and_malformed_markers_fail_closed() {
12673 let current_spec = current_supervisor_bridge_spec();
12674 let current = TrustedPeerDescriptor::try_from(current_spec.clone())
12675 .expect("valid current supervisor");
12676 let target = supervisor_bridge_spec();
12677
12678 let runtime_impl = bootstrap_runtime(
12679 PEER_ID_RECEIVER,
12680 "inproc://raw-rotation-version",
12681 Some("token"),
12682 );
12683 let completed = Arc::clone(&runtime_impl.completed_count);
12684 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
12685 let adapter = Arc::new(MeerkatMachine::ephemeral());
12686 let session_id = SessionId::new();
12687 adapter
12688 .register_session(session_id.clone())
12689 .await
12690 .expect("register session");
12691 adapter
12692 .stage_supervisor_bind(
12693 &session_id,
12694 current.name.to_string(),
12695 current.peer_id.as_str(),
12696 current.address.to_string(),
12697 signing_public_key_for_descriptor(¤t),
12698 1,
12699 )
12700 .await
12701 .expect("bind current supervisor");
12702
12703 let operation_id = SupervisorRotationOperationId::new();
12704 let raw_unsupported = json!({
12705 "delivery": "submit_supervisor_rotation",
12706 "operation_id": operation_id,
12707 "target": target,
12708 "target_epoch": 2,
12709 "protocol_version": 99,
12710 });
12711 let interaction_id = InteractionId(Uuid::new_v4());
12712 let ingress = PeerIngressFact::peer(
12713 interaction_id,
12714 PeerInputClass::ActionableMessage,
12715 meerkat_core::PeerIngressKind::Message,
12716 Some(meerkat_core::PeerIngressAuthDecision::Required),
12717 PeerIngressIdentity::new(
12718 current.peer_id,
12719 current.name.as_str(),
12720 meerkat_core::PeerIngressConvention::Message,
12721 )
12722 .with_signing_pubkey(current.pubkey),
12723 );
12724 let candidate = bridge_delivery_candidate_with_ingress(
12725 current.name.as_str(),
12726 &raw_unsupported,
12727 ingress,
12728 );
12729 assert!(
12730 try_handle_supervisor_bridge_delivery(&adapter, &session_id, &runtime, &candidate,)
12731 .await,
12732 "exact delivery marker must be consumed before typed version decode"
12733 );
12734 let receipt = adapter
12735 .active_supervisor_rotation(&session_id)
12736 .await
12737 .expect("read rejected rotation")
12738 .expect("unsupported rotation is durably recorded");
12739 assert_eq!(
12740 receipt.phase,
12741 crate::meerkat_machine::dsl::SupervisorRotationPhase::Rejected
12742 );
12743 assert_eq!(
12744 receipt.rejection,
12745 Some(
12746 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::UnsupportedProtocolVersion
12747 )
12748 );
12749 assert_eq!(
12750 completed.load(std::sync::atomic::Ordering::SeqCst),
12751 1,
12752 "one-way unsupported submission must close local ingress"
12753 );
12754
12755 let malformed_runtime_impl = bootstrap_runtime(
12756 PEER_ID_RECEIVER,
12757 "inproc://raw-rotation-malformed",
12758 Some("token"),
12759 );
12760 let malformed_completed = Arc::clone(&malformed_runtime_impl.completed_count);
12761 let malformed_runtime: Arc<dyn CommsRuntime> = Arc::new(malformed_runtime_impl);
12762 let malformed_adapter = Arc::new(MeerkatMachine::ephemeral());
12763 let malformed_session = SessionId::new();
12764 malformed_adapter
12765 .register_session(malformed_session.clone())
12766 .await
12767 .expect("register malformed-marker session");
12768 let malformed_id = InteractionId(Uuid::new_v4());
12769 let malformed_ingress = PeerIngressFact::peer(
12770 malformed_id,
12771 PeerInputClass::ActionableMessage,
12772 meerkat_core::PeerIngressKind::Message,
12773 Some(meerkat_core::PeerIngressAuthDecision::Required),
12774 PeerIngressIdentity::new(
12775 current.peer_id,
12776 current.name.as_str(),
12777 meerkat_core::PeerIngressConvention::Message,
12778 )
12779 .with_signing_pubkey(current.pubkey),
12780 );
12781 let malformed = json!({
12782 "delivery": "submit_supervisor_rotation",
12783 "protocol_version": 4,
12784 });
12785 let malformed_candidate = bridge_delivery_candidate_with_ingress(
12786 current.name.as_str(),
12787 &malformed,
12788 malformed_ingress,
12789 );
12790 assert!(
12791 try_handle_supervisor_bridge_delivery(
12792 &malformed_adapter,
12793 &malformed_session,
12794 &malformed_runtime,
12795 &malformed_candidate,
12796 )
12797 .await,
12798 "malformed exact marker must be consumed fail-closed"
12799 );
12800 assert!(
12801 malformed_adapter
12802 .active_supervisor_rotation(&malformed_session)
12803 .await
12804 .expect("read malformed session")
12805 .is_none(),
12806 "malformed marker must not create partial durable operation state"
12807 );
12808 assert_eq!(
12809 malformed_completed.load(std::sync::atomic::Ordering::SeqCst),
12810 1,
12811 "malformed marker must be completed locally instead of reaching the user prompt"
12812 );
12813
12814 let duplicate_base = json!({
12815 "delivery": "submit_supervisor_rotation",
12816 "operation_id": SupervisorRotationOperationId::new(),
12817 "target": supervisor_bridge_spec(),
12818 "target_epoch": 2,
12819 "protocol_version": 4,
12820 })
12821 .to_string();
12822 let duplicate_body = format!(
12823 "{},\"delivery\":\"chat\"}}",
12824 duplicate_base
12825 .strip_suffix('}')
12826 .expect("delivery object has closing brace")
12827 );
12828 let duplicate_id = InteractionId(Uuid::new_v4());
12829 let duplicate_ingress = PeerIngressFact::peer(
12830 duplicate_id,
12831 PeerInputClass::ActionableMessage,
12832 meerkat_core::PeerIngressKind::Message,
12833 Some(meerkat_core::PeerIngressAuthDecision::Required),
12834 PeerIngressIdentity::new(
12835 current.peer_id,
12836 current.name.as_str(),
12837 meerkat_core::PeerIngressConvention::Message,
12838 )
12839 .with_signing_pubkey(current.pubkey),
12840 );
12841 let duplicate_candidate = bridge_delivery_raw_candidate_with_ingress(
12842 current.name.as_str(),
12843 duplicate_body,
12844 duplicate_ingress,
12845 );
12846 assert!(
12847 try_handle_supervisor_bridge_delivery(
12848 &malformed_adapter,
12849 &malformed_session,
12850 &malformed_runtime,
12851 &duplicate_candidate,
12852 )
12853 .await,
12854 "reserved-first duplicate delivery marker must remain reserved and fail closed"
12855 );
12856 assert_eq!(
12857 malformed_completed.load(std::sync::atomic::Ordering::SeqCst),
12858 2,
12859 "duplicate/conflicting delivery keys must be consumed instead of reaching the user prompt"
12860 );
12861 assert!(
12862 malformed_adapter
12863 .active_supervisor_rotation(&malformed_session)
12864 .await
12865 .expect("read duplicate-marker session")
12866 .is_none(),
12867 "duplicate marker must not create durable operation state"
12868 );
12869 }
12870
12871 #[tokio::test]
12872 async fn pending_rotation_observe_does_not_mint_temporary_supervisor_trust() {
12873 let previous_spec = current_supervisor_bridge_spec();
12874 let previous = TrustedPeerDescriptor::try_from(previous_spec.clone())
12875 .expect("valid previous supervisor");
12876 let next_spec = supervisor_bridge_spec();
12877 let runtime_impl = bootstrap_runtime(
12878 PEER_ID_RECEIVER,
12879 "inproc://pending-observe-no-retrust",
12880 Some("token"),
12881 );
12882 let trusted_peer_ids = Arc::clone(&runtime_impl.trusted_peer_ids);
12883 let sent_commands = Arc::clone(&runtime_impl.sent_commands);
12884 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
12885 let adapter = Arc::new(MeerkatMachine::ephemeral());
12886 let session_id = SessionId::new();
12887 adapter
12888 .register_session(session_id.clone())
12889 .await
12890 .expect("register session");
12891 adapter
12892 .stage_supervisor_bind(
12893 &session_id,
12894 previous.name.to_string(),
12895 previous.peer_id.as_str(),
12896 previous.address.to_string(),
12897 signing_public_key_for_descriptor(&previous),
12898 1,
12899 )
12900 .await
12901 .expect("bind previous supervisor");
12902
12903 let operation_id = SupervisorRotationOperationId::new();
12904 let next =
12905 TrustedPeerDescriptor::try_from(next_spec.clone()).expect("valid next supervisor");
12906 let submission = adapter
12907 .submit_supervisor_rotation(
12908 &session_id,
12909 GeneratedSupervisorRotationSubmit {
12910 operation_id: operation_id.to_string(),
12911 next: GeneratedSupervisorBinding {
12912 name: next.name.to_string(),
12913 peer_id: next.peer_id.as_str(),
12914 address: next.address.to_string(),
12915 signing_public_key: signing_public_key_for_descriptor(&next),
12916 epoch: 2,
12917 },
12918 preflight_rejection: None,
12919 sender_peer_id: Some(previous.peer_id.as_str()),
12920 sender_signing_public_key: Some(signing_public_key_for_descriptor(&previous)),
12921 },
12922 )
12923 .await
12924 .expect("submit pending rotation");
12925 assert!(matches!(submission, SupervisorRotationSubmission::New(_)));
12926
12927 let command = BridgeCommand::ObserveSupervisorRotation(
12928 meerkat_contracts::wire::supervisor_bridge::BridgeSupervisorRotationObserve {
12929 operation_id,
12930 observer: previous_spec,
12931 observer_epoch: 1,
12932 protocol_version: BridgeProtocolVersion::V4,
12933 },
12934 );
12935 let interaction_id = InteractionId(Uuid::new_v4());
12936 let ingress = PeerIngressFact::peer(
12937 interaction_id,
12938 PeerInputClass::ActionableRequest,
12939 meerkat_core::PeerIngressKind::Request,
12940 Some(meerkat_core::PeerIngressAuthDecision::Required),
12941 PeerIngressIdentity::new(
12942 previous.peer_id,
12943 previous.name.as_str(),
12944 meerkat_core::PeerIngressConvention::Request {
12945 request_id: interaction_id.to_string(),
12946 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
12947 },
12948 )
12949 .with_signing_pubkey(previous.pubkey),
12950 );
12951 let candidate = bridge_candidate_with_ingress(previous.name.as_str(), &command, ingress);
12952 assert!(
12953 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate).await
12954 );
12955 assert!(
12956 trusted_peer_ids.lock().await.is_empty(),
12957 "pending observation must not temporarily retrust either rotation participant"
12958 );
12959 let replies = recorded_bridge_replies(&sent_commands).await;
12960 assert!(matches!(
12961 replies.as_slice(),
12962 [(
12963 meerkat_core::interaction::ResponseStatus::Completed,
12964 BridgeReply::SupervisorRotation(BridgeSupervisorRotationObservation::Found {
12965 state: BridgeSupervisorRotationState::Pending { .. }
12966 })
12967 )]
12968 ));
12969 }
12970
12971 #[tokio::test]
12972 async fn replacement_runtime_hydrates_completed_and_rejected_rotation_authority() {
12973 let previous = trusted_supervisor_descriptor(0xcc);
12974 let next = trusted_supervisor_descriptor(0xbb);
12975
12976 let completed_runtime_name = format!("completed-rotation-hydration-{}", Uuid::new_v4());
12977 let completed_runtime = Arc::new(
12978 meerkat_comms::CommsRuntime::inproc_only(&completed_runtime_name)
12979 .expect("completed replacement runtime"),
12980 );
12981 let completed_runtime_dyn: Arc<dyn CommsRuntime> = completed_runtime.clone();
12982 let completed_adapter = Arc::new(MeerkatMachine::ephemeral());
12983 let completed_session = SessionId::new();
12984 completed_adapter
12985 .register_session(completed_session.clone())
12986 .await
12987 .expect("register completed session");
12988 completed_adapter
12989 .stage_supervisor_bind(
12990 &completed_session,
12991 previous.name.to_string(),
12992 previous.peer_id.as_str(),
12993 previous.address.to_string(),
12994 signing_public_key_for_descriptor(&previous),
12995 1,
12996 )
12997 .await
12998 .expect("bind previous completed supervisor");
12999 completed_adapter
13000 .test_install_session_peer_comms_handle_on_runtime(
13001 &completed_session,
13002 completed_runtime.as_ref(),
13003 )
13004 .await
13005 .expect("install completed runtime owner token");
13006 completed_adapter
13007 .stage_local_endpoint_for_comms_runtime(
13008 &completed_session,
13009 completed_runtime_dyn.as_ref(),
13010 )
13011 .await
13012 .expect("stage completed runtime local endpoint");
13013
13014 let completed_operation = SupervisorRotationOperationId::new();
13015 completed_adapter
13016 .submit_supervisor_rotation(
13017 &completed_session,
13018 GeneratedSupervisorRotationSubmit {
13019 operation_id: completed_operation.to_string(),
13020 next: GeneratedSupervisorBinding {
13021 name: next.name.to_string(),
13022 peer_id: next.peer_id.as_str(),
13023 address: next.address.to_string(),
13024 signing_public_key: signing_public_key_for_descriptor(&next),
13025 epoch: 2,
13026 },
13027 preflight_rejection: None,
13028 sender_peer_id: Some(previous.peer_id.as_str()),
13029 sender_signing_public_key: Some(signing_public_key_for_descriptor(&previous)),
13030 },
13031 )
13032 .await
13033 .expect("submit completed rotation");
13034 completed_adapter
13035 .stage_supervisor_rotation_resume(&completed_session, completed_operation.to_string())
13036 .await
13037 .expect("resume completed rotation");
13038 completed_adapter
13039 .stage_supervisor_rotation_previous_revoked(
13040 &completed_session,
13041 completed_operation.to_string(),
13042 previous.peer_id.as_str(),
13043 1,
13044 )
13045 .await
13046 .expect("checkpoint previous revoke");
13047 completed_adapter
13048 .stage_supervisor_rotation_next_published(
13049 &completed_session,
13050 completed_operation.to_string(),
13051 next.peer_id.as_str(),
13052 2,
13053 )
13054 .await
13055 .expect("checkpoint next publish");
13056 assert!(
13057 !completed_runtime
13058 .router()
13059 .is_private(&meerkat_comms::PubKey::new(next.pubkey))
13060 );
13061 assert!(
13062 hydrate_current_bound_supervisor_route(
13063 &completed_adapter,
13064 &completed_session,
13065 &completed_runtime_dyn,
13066 )
13067 .await
13068 .expect("hydrate completed authority")
13069 );
13070 assert!(
13071 completed_runtime
13072 .router()
13073 .is_private(&meerkat_comms::PubKey::new(next.pubkey)),
13074 "Completed replacement runtime must hydrate only the generated next authority"
13075 );
13076 assert!(
13077 !completed_runtime
13078 .router()
13079 .is_private(&meerkat_comms::PubKey::new(previous.pubkey)),
13080 "Completed replacement runtime must not hydrate the revoked authority"
13081 );
13082
13083 let rejected_runtime_name = format!("rejected-rotation-hydration-{}", Uuid::new_v4());
13084 let rejected_runtime = Arc::new(
13085 meerkat_comms::CommsRuntime::inproc_only(&rejected_runtime_name)
13086 .expect("rejected replacement runtime"),
13087 );
13088 let rejected_runtime_dyn: Arc<dyn CommsRuntime> = rejected_runtime.clone();
13089 let rejected_adapter = Arc::new(MeerkatMachine::ephemeral());
13090 let rejected_session = SessionId::new();
13091 rejected_adapter
13092 .register_session(rejected_session.clone())
13093 .await
13094 .expect("register rejected session");
13095 rejected_adapter
13096 .stage_supervisor_bind(
13097 &rejected_session,
13098 previous.name.to_string(),
13099 previous.peer_id.as_str(),
13100 previous.address.to_string(),
13101 signing_public_key_for_descriptor(&previous),
13102 1,
13103 )
13104 .await
13105 .expect("bind previous rejected supervisor");
13106 rejected_adapter
13107 .test_install_session_peer_comms_handle_on_runtime(
13108 &rejected_session,
13109 rejected_runtime.as_ref(),
13110 )
13111 .await
13112 .expect("install rejected runtime owner token");
13113 rejected_adapter
13114 .stage_local_endpoint_for_comms_runtime(
13115 &rejected_session,
13116 rejected_runtime_dyn.as_ref(),
13117 )
13118 .await
13119 .expect("stage rejected runtime local endpoint");
13120 rejected_adapter
13121 .submit_supervisor_rotation(
13122 &rejected_session,
13123 GeneratedSupervisorRotationSubmit {
13124 operation_id: SupervisorRotationOperationId::new().to_string(),
13125 next: GeneratedSupervisorBinding {
13126 name: next.name.to_string(),
13127 peer_id: next.peer_id.as_str(),
13128 address: next.address.to_string(),
13129 signing_public_key: signing_public_key_for_descriptor(&next),
13130 epoch: 2,
13131 },
13132 preflight_rejection: Some(
13133 crate::meerkat_machine::dsl::SupervisorRotationRejectionKind::UnsupportedProtocolVersion,
13134 ),
13135 sender_peer_id: Some(previous.peer_id.as_str()),
13136 sender_signing_public_key: Some(signing_public_key_for_descriptor(&previous)),
13137 },
13138 )
13139 .await
13140 .expect("durably reject rotation");
13141 assert!(
13142 hydrate_current_bound_supervisor_route(
13143 &rejected_adapter,
13144 &rejected_session,
13145 &rejected_runtime_dyn,
13146 )
13147 .await
13148 .expect("hydrate rejected authority")
13149 );
13150 assert!(
13151 rejected_runtime
13152 .router()
13153 .is_private(&meerkat_comms::PubKey::new(previous.pubkey)),
13154 "Rejected replacement runtime must hydrate only the retained previous authority"
13155 );
13156 assert!(
13157 !rejected_runtime
13158 .router()
13159 .is_private(&meerkat_comms::PubKey::new(next.pubkey)),
13160 "Rejected replacement runtime must not hydrate the rejected target"
13161 );
13162 }
13163
13164 #[tokio::test]
13165 async fn bind_member_rolls_back_binding_when_trust_publication_fails() {
13166 let payload = sample_bind_payload();
13169 let sender = payload.supervisor.peer_id.clone();
13170 let mut runtime_impl = bootstrap_runtime(
13171 PEER_ID_RECEIVER,
13172 "inproc://receiver",
13173 Some("expected-token"),
13174 );
13175 runtime_impl
13176 .add_trusted_peer_errors
13177 .insert(payload.supervisor.peer_id.clone(), "boom".to_string());
13178 let trusted_peer_ids = runtime_impl.trusted_peer_ids.clone();
13179 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
13180 let adapter = Arc::new(MeerkatMachine::ephemeral());
13181 let session_id = SessionId::new();
13182 adapter
13183 .register_session(session_id.clone())
13184 .await
13185 .expect("register session");
13186 let candidate = bridge_candidate(&sender, &BridgeCommand::BindMember(payload));
13187
13188 assert!(
13189 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate).await
13190 );
13191 assert!(
13192 matches!(
13193 adapter.supervisor_binding(&session_id).await,
13194 SupervisorBinding::Unbound
13195 ),
13196 "failed trust publication must roll BindMember back to Unbound"
13197 );
13198 assert!(
13199 trusted_peer_ids.lock().await.is_empty(),
13200 "failed trust publication must not leave the supervisor trusted"
13201 );
13202 }
13203
13204 #[tokio::test]
13205 async fn bind_member_invalid_bootstrap_does_not_publish_response_route() {
13206 let mut payload = sample_bind_payload();
13207 payload.bootstrap_token = "wrong-token".into();
13208 let sender = payload.supervisor.peer_id.clone();
13209 let runtime_impl = bootstrap_runtime(
13210 PEER_ID_RECEIVER,
13211 "inproc://receiver",
13212 Some("expected-token"),
13213 );
13214 let trusted_peer_ids = runtime_impl.trusted_peer_ids.clone();
13215 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
13216 let adapter = Arc::new(MeerkatMachine::ephemeral());
13217 let session_id = SessionId::new();
13218 adapter
13219 .register_session(session_id.clone())
13220 .await
13221 .expect("register session");
13222 let candidate = bridge_candidate(&sender, &BridgeCommand::BindMember(payload));
13223
13224 assert!(
13225 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate).await
13226 );
13227 assert!(
13228 matches!(
13229 adapter.supervisor_binding(&session_id).await,
13230 SupervisorBinding::Unbound
13231 ),
13232 "invalid bootstrap must not bind supervisor authority"
13233 );
13234 assert!(
13235 trusted_peer_ids.lock().await.is_empty(),
13236 "invalid bootstrap must not temporarily publish supervisor trust just to route rejection"
13237 );
13238 }
13239
13240 #[tokio::test]
13241 async fn revoke_supervisor_failure_leaves_pending_checkpoint_and_retry_completes() {
13242 let supervisor = supervisor_bridge_spec();
13243 let runtime = Arc::new(bootstrap_runtime(
13244 PEER_ID_RECEIVER,
13245 "inproc://receiver",
13246 Some("expected-token"),
13247 ));
13248 runtime
13249 .remove_failures_remaining
13250 .store(1, std::sync::atomic::Ordering::SeqCst);
13251 let runtime_dyn: Arc<dyn CommsRuntime> = Arc::clone(&runtime) as Arc<dyn CommsRuntime>;
13252 let adapter = Arc::new(MeerkatMachine::ephemeral());
13253 let session_id = SessionId::new();
13254 adapter
13255 .register_session(session_id.clone())
13256 .await
13257 .expect("register session");
13258 let payload = BridgeSupervisorPayload {
13259 supervisor: supervisor.clone(),
13260 epoch: 1,
13261 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
13262 };
13263 let candidate = bridge_candidate(
13264 &supervisor.peer_id,
13265 &BridgeCommand::RevokeSupervisor(payload.clone()),
13266 );
13267 let spec =
13268 TrustedPeerDescriptor::try_from(supervisor.clone()).expect("valid supervisor spec");
13269 adapter
13270 .stage_supervisor_bind(
13271 &session_id,
13272 spec.name.to_string(),
13273 spec.peer_id.as_str(),
13274 spec.address.to_string(),
13275 signing_public_key_for_descriptor(&spec),
13276 payload.epoch,
13277 )
13278 .await
13279 .expect("pre-bind supervisor");
13280
13281 assert!(
13282 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime_dyn, &candidate,)
13283 .await,
13284 "revoke command should be handled"
13285 );
13286 assert!(
13287 matches!(
13288 adapter.supervisor_binding(&session_id).await,
13289 SupervisorBinding::Unbound
13290 ),
13291 "failed removal must leave durable revoke pending, not restore a live binding"
13292 );
13293 let retry = bridge_candidate(
13294 &supervisor.peer_id,
13295 &BridgeCommand::RevokeSupervisor(payload),
13296 );
13297 assert!(
13298 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime_dyn, &retry).await,
13299 "exact revoke retry should complete the pending checkpoint"
13300 );
13301 assert!(
13302 runtime
13303 .remove_attempts
13304 .load(std::sync::atomic::Ordering::SeqCst)
13305 >= 2,
13306 "retry must re-drive generated trust removal with fresh authority"
13307 );
13308 }
13309
13310 #[tokio::test]
13317 async fn dsl_supervisor_guards_block_rebind_and_stale_revoke() {
13318 let adapter = Arc::new(MeerkatMachine::ephemeral());
13319 let session_id = SessionId::new();
13320 adapter
13321 .register_session(session_id.clone())
13322 .await
13323 .expect("register session");
13324
13325 adapter
13327 .stage_supervisor_bind(
13328 &session_id,
13329 "super-a".to_string(),
13330 "ed25519:super-a".to_string(),
13331 "inproc://super-a".to_string(),
13332 test_supervisor_signing_public_key(0xaa),
13333 1,
13334 )
13335 .await
13336 .expect("initial bind");
13337
13338 let rebind = adapter
13342 .stage_supervisor_bind(
13343 &session_id,
13344 "super-b".to_string(),
13345 "ed25519:super-b".to_string(),
13346 "inproc://super-b".to_string(),
13347 test_supervisor_signing_public_key(0xbb),
13348 2,
13349 )
13350 .await;
13351 assert!(
13352 matches!(rebind, Err(SupervisorBindingStageError::Dsl(_))),
13353 "double bind must be rejected by DSL guard, got: {rebind:?}"
13354 );
13355
13356 let stale_peer_revoke = adapter
13358 .stage_supervisor_revoke(&session_id, "ed25519:super-b".to_string(), 1)
13359 .await;
13360 assert!(
13361 matches!(stale_peer_revoke, Err(SupervisorBindingStageError::Dsl(_))),
13362 "revoke with mismatched peer_id must be DSL-rejected, got: {stale_peer_revoke:?}"
13363 );
13364
13365 let stale_epoch_revoke = adapter
13367 .stage_supervisor_revoke(&session_id, "ed25519:super-a".to_string(), 99)
13368 .await;
13369 assert!(
13370 matches!(stale_epoch_revoke, Err(SupervisorBindingStageError::Dsl(_))),
13371 "revoke with mismatched epoch must be DSL-rejected, got: {stale_epoch_revoke:?}"
13372 );
13373
13374 adapter
13376 .stage_supervisor_authorize(
13377 &session_id,
13378 "super-a-refreshed".to_string(),
13379 "ed25519:super-a".to_string(),
13380 "inproc://super-a-refreshed".to_string(),
13381 test_supervisor_signing_public_key(0xaa),
13382 2,
13383 )
13384 .await
13385 .expect("same-authority version advance must succeed");
13386
13387 match adapter.supervisor_binding(&session_id).await {
13389 SupervisorBinding::Bound { peer_id, epoch, .. } => {
13390 assert_eq!(peer_id, "ed25519:super-a");
13391 assert_eq!(epoch, 2);
13392 }
13393 SupervisorBinding::Unbound => panic!("expected Bound after rotation"),
13394 }
13395
13396 adapter
13399 .stage_supervisor_revoke(&session_id, "ed25519:super-a".to_string(), 2)
13400 .await
13401 .expect("matching revoke must succeed");
13402 assert!(
13403 matches!(
13404 adapter.supervisor_binding(&session_id).await,
13405 SupervisorBinding::Unbound
13406 ),
13407 "binding must be Unbound after matching revoke"
13408 );
13409 }
13410
13411 #[tokio::test]
13418 async fn dsl_supervisor_trust_publish_ack_stale_epoch_is_rejected() {
13419 let adapter = Arc::new(MeerkatMachine::ephemeral());
13420 let session_id = SessionId::new();
13421 adapter
13422 .register_session(session_id.clone())
13423 .await
13424 .expect("register session");
13425
13426 adapter
13428 .stage_supervisor_bind(
13429 &session_id,
13430 "super-a".to_string(),
13431 "ed25519:super-a".to_string(),
13432 "inproc://super-a".to_string(),
13433 test_supervisor_signing_public_key(0xaa),
13434 1,
13435 )
13436 .await
13437 .expect("initial bind");
13438
13439 adapter
13441 .stage_supervisor_authorize(
13442 &session_id,
13443 "super-a".to_string(),
13444 "ed25519:super-a".to_string(),
13445 "inproc://super-a".to_string(),
13446 test_supervisor_signing_public_key(0xaa),
13447 2,
13448 )
13449 .await
13450 .expect("rotation to epoch 2");
13451
13452 let stale = adapter
13456 .stage_supervisor_trust_published(&session_id, "ed25519:super-a".to_string(), 1)
13457 .await;
13458 assert!(
13459 matches!(stale, Err(SupervisorBindingStageError::Dsl(_))),
13460 "stale-epoch publish ack must be DSL-rejected, got: {stale:?}"
13461 );
13462 match adapter.supervisor_binding(&session_id).await {
13464 SupervisorBinding::Bound { epoch, .. } => {
13465 assert_eq!(epoch, 2, "binding must still be at epoch 2");
13466 }
13467 SupervisorBinding::Unbound => panic!("stale ack must not unbind"),
13468 }
13469
13470 adapter
13473 .stage_supervisor_trust_published(&session_id, "ed25519:super-a".to_string(), 2)
13474 .await
13475 .expect("matching-epoch ack must be accepted");
13476 }
13477
13478 #[tokio::test]
13479 async fn generated_supervisor_publish_authority_rejects_stale_descriptor_same_peer_epoch() {
13480 let adapter = Arc::new(MeerkatMachine::ephemeral());
13481 let session_id = SessionId::new();
13482 adapter
13483 .register_session(session_id.clone())
13484 .await
13485 .expect("register session");
13486 let runtime = bootstrap_runtime(PEER_ID_RECEIVER, "inproc://receiver", Some("token"));
13487 adapter
13488 .stage_local_endpoint_for_comms_runtime(&session_id, &runtime)
13489 .await
13490 .expect("stage local endpoint");
13491 let supervisor_peer_id = test_pubkey(0xaa).to_peer_id().as_str();
13492
13493 let transition = adapter
13494 .stage_supervisor_bind(
13495 &session_id,
13496 "super-a".to_string(),
13497 supervisor_peer_id.clone(),
13498 "inproc://super-old".to_string(),
13499 test_supervisor_signing_public_key(0xaa),
13500 7,
13501 )
13502 .await
13503 .expect("initial bind");
13504 let freshness = adapter
13505 .supervisor_trust_publish_freshness_authority(&session_id)
13506 .await
13507 .expect("publish freshness");
13508 let stale_obligation =
13509 crate::protocol_supervisor_trust_publish::extract_obligations_with_freshness(
13510 &transition,
13511 freshness,
13512 )
13513 .into_iter()
13514 .next()
13515 .expect("generated publish obligation");
13516
13517 adapter
13518 .stage_supervisor_binding_route_refresh(
13519 &session_id,
13520 GeneratedSupervisorBinding {
13521 name: "super-a-renamed".to_string(),
13522 peer_id: supervisor_peer_id.clone(),
13523 address: "inproc://super-new".to_string(),
13524 signing_public_key: test_supervisor_signing_public_key(0xaa),
13525 epoch: 7,
13526 },
13527 Some(supervisor_peer_id),
13528 )
13529 .await
13530 .expect("same-authority same-epoch route refresh");
13531
13532 let stale_authority = crate::protocol_supervisor_trust_publish::publish_authority_for_peer(
13533 &stale_obligation,
13534 stale_obligation.peer_id(),
13535 );
13536 assert!(
13537 matches!(stale_authority, Err(ref error) if error.contains("stale generated supervisor trust publish obligation")),
13538 "stale descriptor must not mint generated supervisor publish authority, got: {stale_authority:?}"
13539 );
13540 }
13541
13542 #[tokio::test]
13543 async fn dsl_supervisor_trust_revoke_ack_stale_epoch_is_rejected() {
13544 let adapter = Arc::new(MeerkatMachine::ephemeral());
13545 let session_id = SessionId::new();
13546 adapter
13547 .register_session(session_id.clone())
13548 .await
13549 .expect("register session");
13550
13551 adapter
13553 .stage_supervisor_bind(
13554 &session_id,
13555 "super-a".to_string(),
13556 "ed25519:super-a".to_string(),
13557 "inproc://super-a".to_string(),
13558 test_supervisor_signing_public_key(0xaa),
13559 1,
13560 )
13561 .await
13562 .expect("initial bind");
13563
13564 adapter
13566 .stage_supervisor_authorize(
13567 &session_id,
13568 "super-a".to_string(),
13569 "ed25519:super-a".to_string(),
13570 "inproc://super-a".to_string(),
13571 test_supervisor_signing_public_key(0xaa),
13572 2,
13573 )
13574 .await
13575 .expect("rotation to epoch 2");
13576
13577 adapter
13580 .stage_supervisor_revoke(&session_id, "ed25519:super-a".to_string(), 2)
13581 .await
13582 .expect("matching revoke must stage pending trust revoke");
13583
13584 let stale = adapter
13587 .stage_supervisor_trust_revoked(&session_id, "ed25519:super-a".to_string(), 1)
13588 .await;
13589 assert!(
13590 matches!(stale, Err(SupervisorBindingStageError::Dsl(_))),
13591 "stale-epoch revoke ack must be DSL-rejected, got: {stale:?}"
13592 );
13593 match adapter.supervisor_binding(&session_id).await {
13594 SupervisorBinding::Unbound => {}
13595 SupervisorBinding::Bound { .. } => panic!("revoke input must unbind before feedback"),
13596 }
13597
13598 adapter
13600 .stage_supervisor_trust_revoked(&session_id, "ed25519:super-a".to_string(), 2)
13601 .await
13602 .expect("matching-epoch revoke ack must be accepted");
13603 }
13604
13605 #[tokio::test]
13606 async fn dsl_supervisor_trust_ack_rejects_mismatched_peer_id() {
13607 let adapter = Arc::new(MeerkatMachine::ephemeral());
13608 let session_id = SessionId::new();
13609 adapter
13610 .register_session(session_id.clone())
13611 .await
13612 .expect("register session");
13613
13614 adapter
13615 .stage_supervisor_bind(
13616 &session_id,
13617 "super-a".to_string(),
13618 "ed25519:super-a".to_string(),
13619 "inproc://super-a".to_string(),
13620 test_supervisor_signing_public_key(0xaa),
13621 7,
13622 )
13623 .await
13624 .expect("initial bind");
13625
13626 let wrong_peer = adapter
13628 .stage_supervisor_trust_published(&session_id, "ed25519:unrelated".to_string(), 7)
13629 .await;
13630 assert!(
13631 matches!(wrong_peer, Err(SupervisorBindingStageError::Dsl(_))),
13632 "mismatched-peer ack must be DSL-rejected, got: {wrong_peer:?}"
13633 );
13634 }
13635
13636 #[tokio::test]
13637 async fn dsl_supervisor_trust_ack_rejected_when_unbound() {
13638 let adapter = Arc::new(MeerkatMachine::ephemeral());
13639 let session_id = SessionId::new();
13640 adapter
13641 .register_session(session_id.clone())
13642 .await
13643 .expect("register session");
13644
13645 let unbound = adapter
13647 .stage_supervisor_trust_published(&session_id, "ed25519:super-a".to_string(), 1)
13648 .await;
13649 assert!(
13650 matches!(unbound, Err(SupervisorBindingStageError::Dsl(_))),
13651 "publish ack must be rejected when Unbound, got: {unbound:?}"
13652 );
13653 }
13654
13655 #[tokio::test]
13656 async fn wire_member_rejects_invalid_peer_spec_before_trusting_it() {
13657 let runtime = Arc::new(meerkat_comms::CommsRuntime::inproc_only("receiver-wire").unwrap());
13658 let supervisor_runtime =
13659 Arc::new(meerkat_comms::CommsRuntime::inproc_only("mob/__mob_supervisor__").unwrap());
13660 let adapter = Arc::new(MeerkatMachine::ephemeral());
13661 let session_id = SessionId::new();
13662 adapter
13663 .register_session(session_id.clone())
13664 .await
13665 .expect("register session");
13666 let supervisor = trusted_peer_from_runtime("mob/__mob_supervisor__", &supervisor_runtime);
13667 add_test_projection_trust(runtime.as_ref(), supervisor.clone(), "trust supervisor").await;
13668 adapter
13669 .stage_supervisor_bind(
13670 &session_id,
13671 supervisor.name.to_string(),
13672 supervisor.peer_id.as_str(),
13673 supervisor.address.to_string(),
13674 signing_public_key_for_descriptor(&supervisor),
13675 1,
13676 )
13677 .await
13678 .expect("pre-bind supervisor");
13679 let invalid_peer_id = PeerId::new().as_str();
13680 let candidate = bridge_candidate(
13681 &supervisor.peer_id.as_str(),
13682 &BridgeCommand::WireMember(BridgePeerWiringPayload {
13683 supervisor: supervisor.clone().into(),
13684 epoch: 1,
13685 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
13686 peer_spec: BridgePeerSpec {
13687 name: "".to_string(),
13688 peer_id: invalid_peer_id.clone(),
13689 address: "inproc://peer".to_string(),
13690 pubkey: [0u8; 32],
13691 },
13692 mob_peer_overlay: Some(BridgeMobPeerOverlayHandoff {
13693 recipient_peer_id: runtime.peer_id().expect("runtime peer_id").as_str(),
13694 topology_epoch: 1,
13695 peer_specs: Vec::new(),
13696 }),
13697 }),
13698 );
13699
13700 assert!(
13701 try_handle_supervisor_bridge_command(
13702 &adapter,
13703 &session_id,
13704 &(runtime.clone() as Arc<dyn CommsRuntime>),
13705 &candidate,
13706 )
13707 .await,
13708 "wire bridge command should be handled"
13709 );
13710 let peers = runtime.peers().await;
13711 assert!(
13712 peers.iter().all(|entry| !entry.name.as_str().is_empty()),
13713 "invalid wire peer specs must not be materialized in comms trust"
13714 );
13715 }
13716
13717 #[tokio::test]
13718 async fn wire_member_without_overlay_rejects_v2_payload_without_wiring() {
13719 let runtime =
13725 Arc::new(meerkat_comms::CommsRuntime::inproc_only("receiver-wire-noverlay").unwrap());
13726 let supervisor_runtime = Arc::new(
13727 meerkat_comms::CommsRuntime::inproc_only("mob/__mob_supervisor_noverlay__").unwrap(),
13728 );
13729 let peer_runtime =
13730 Arc::new(meerkat_comms::CommsRuntime::inproc_only("peer-wire-noverlay").unwrap());
13731 let adapter = Arc::new(MeerkatMachine::ephemeral());
13732 let session_id = SessionId::new();
13733 adapter
13734 .register_session(session_id.clone())
13735 .await
13736 .expect("register session");
13737 let supervisor =
13738 trusted_peer_from_runtime("mob/__mob_supervisor_noverlay__", &supervisor_runtime);
13739 add_test_projection_trust(runtime.as_ref(), supervisor.clone(), "trust supervisor").await;
13740 adapter
13741 .stage_supervisor_bind(
13742 &session_id,
13743 supervisor.name.to_string(),
13744 supervisor.peer_id.as_str(),
13745 supervisor.address.to_string(),
13746 signing_public_key_for_descriptor(&supervisor),
13747 1,
13748 )
13749 .await
13750 .expect("pre-bind supervisor");
13751 let peer_spec = trusted_peer_from_runtime("peer-wire-noverlay", &peer_runtime);
13752 let candidate = bridge_candidate(
13753 &supervisor.peer_id.as_str(),
13754 &BridgeCommand::WireMember(BridgePeerWiringPayload {
13755 supervisor: supervisor.clone().into(),
13756 epoch: 1,
13757 protocol_version: BridgeProtocolVersion::V2,
13758 peer_spec: peer_spec.clone().into(),
13759 mob_peer_overlay: None,
13760 }),
13761 );
13762
13763 assert!(
13764 try_handle_supervisor_bridge_command(
13765 &adapter,
13766 &session_id,
13767 &(runtime.clone() as Arc<dyn CommsRuntime>),
13768 &candidate,
13769 )
13770 .await,
13771 "wire bridge command should be handled (and cleanly rejected)"
13772 );
13773 let peers = runtime.peers().await;
13774 assert!(
13775 peers.iter().all(|entry| entry.peer_id != peer_spec.peer_id),
13776 "a V2 wiring payload without the overlay must not materialize peer trust"
13777 );
13778 }
13779
13780 #[tokio::test]
13781 async fn declare_member_outbound_taint_from_bound_supervisor_installs_declaration() {
13782 let runtime =
13783 Arc::new(meerkat_comms::CommsRuntime::inproc_only("receiver-taint-install").unwrap());
13784 install_test_peer_request_response_authority(
13787 &runtime,
13788 Arc::new(CountingPeerInteractionHandle::default()),
13789 );
13790 let supervisor_runtime = Arc::new(
13791 meerkat_comms::CommsRuntime::inproc_only("mob/__mob_supervisor_taint__").unwrap(),
13792 );
13793 let adapter = Arc::new(MeerkatMachine::ephemeral());
13794 let session_id = SessionId::new();
13795 adapter
13796 .register_session(session_id.clone())
13797 .await
13798 .expect("register session");
13799 let supervisor =
13800 trusted_peer_from_runtime("mob/__mob_supervisor_taint__", &supervisor_runtime);
13801 adapter
13802 .stage_supervisor_bind(
13803 &session_id,
13804 supervisor.name.to_string(),
13805 supervisor.peer_id.as_str(),
13806 supervisor.address.to_string(),
13807 signing_public_key_for_descriptor(&supervisor),
13808 1,
13809 )
13810 .await
13811 .expect("pre-bind supervisor");
13812 adapter
13815 .test_install_session_peer_comms_handle_on_runtime(&session_id, runtime.as_ref())
13816 .await
13817 .expect("install adapter session peer-comms handle on member runtime");
13818
13819 let command = BridgeCommand::DeclareMemberOutboundTaint(BridgeOutboundTaintPayload {
13820 supervisor: supervisor.clone().into(),
13821 epoch: 1,
13822 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
13823 target: Some(BridgeOutboundTaintTarget::PeerOnly),
13824 taint: Some(meerkat_core::comms::SenderContentTaint::Tainted),
13825 });
13826 let interaction_id = InteractionId(Uuid::new_v4());
13827 let ingress = PeerIngressFact::peer(
13828 interaction_id,
13829 PeerInputClass::ActionableRequest,
13830 meerkat_core::PeerIngressKind::Request,
13831 Some(meerkat_core::PeerIngressAuthDecision::Required),
13832 PeerIngressIdentity::new(
13833 supervisor.peer_id,
13834 supervisor.name.as_str(),
13835 meerkat_core::PeerIngressConvention::Request {
13836 request_id: interaction_id.to_string(),
13837 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
13838 },
13839 )
13840 .with_signing_pubkey(supervisor.pubkey),
13841 );
13842 let candidate = bridge_candidate_with_ingress(supervisor.name.as_str(), &command, ingress);
13843
13844 assert!(
13845 try_handle_supervisor_bridge_command(
13846 &adapter,
13847 &session_id,
13848 &(runtime.clone() as Arc<dyn CommsRuntime>),
13849 &candidate,
13850 )
13851 .await,
13852 "declare-outbound-taint bridge command should be handled"
13853 );
13854 assert_eq!(
13855 runtime.outbound_content_taint(),
13856 Some(meerkat_core::comms::SenderContentTaint::Tainted),
13857 "the relayed host declaration must install on the member runtime"
13858 );
13859
13860 runtime.set_outbound_content_taint(None);
13861 let legacy_command =
13862 BridgeCommand::DeclareMemberOutboundTaint(BridgeOutboundTaintPayload {
13863 supervisor: supervisor.clone().into(),
13864 epoch: 1,
13865 protocol_version: BridgeProtocolVersion::V3,
13866 target: None,
13867 taint: Some(meerkat_core::comms::SenderContentTaint::Tainted),
13868 });
13869 let legacy_id = InteractionId(Uuid::new_v4());
13870 let legacy_ingress = PeerIngressFact::peer(
13871 legacy_id,
13872 PeerInputClass::ActionableRequest,
13873 meerkat_core::PeerIngressKind::Request,
13874 Some(meerkat_core::PeerIngressAuthDecision::Required),
13875 PeerIngressIdentity::new(
13876 supervisor.peer_id,
13877 supervisor.name.as_str(),
13878 meerkat_core::PeerIngressConvention::Request {
13879 request_id: legacy_id.to_string(),
13880 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
13881 },
13882 )
13883 .with_signing_pubkey(supervisor.pubkey),
13884 );
13885 let legacy_candidate = bridge_candidate_with_ingress(
13886 supervisor.name.as_str(),
13887 &legacy_command,
13888 legacy_ingress,
13889 );
13890 assert!(
13891 try_handle_supervisor_bridge_command(
13892 &adapter,
13893 &session_id,
13894 &(runtime.clone() as Arc<dyn CommsRuntime>),
13895 &legacy_candidate,
13896 )
13897 .await
13898 );
13899 assert_eq!(
13900 runtime.outbound_content_taint(),
13901 Some(meerkat_core::comms::SenderContentTaint::Tainted),
13902 "an omitted-target V3 declaration remains valid for a truly peer-only session"
13903 );
13904 }
13905
13906 #[tokio::test]
13907 async fn delayed_g1_outbound_taint_cannot_mutate_revived_g2_member() {
13908 let runtime =
13909 Arc::new(meerkat_comms::CommsRuntime::inproc_only("receiver-taint-host-g2").unwrap());
13910 install_test_peer_request_response_authority(
13911 &runtime,
13912 Arc::new(CountingPeerInteractionHandle::default()),
13913 );
13914 let supervisor_runtime = Arc::new(
13915 meerkat_comms::CommsRuntime::inproc_only("mob/__mob_supervisor_taint_g2__").unwrap(),
13916 );
13917 let adapter = Arc::new(MeerkatMachine::ephemeral());
13918 let session_id = SessionId::new();
13919 adapter
13920 .register_session(session_id.clone())
13921 .await
13922 .expect("register session");
13923 let supervisor =
13924 trusted_peer_from_runtime("mob/__mob_supervisor_taint_g2__", &supervisor_runtime);
13925 adapter
13926 .stage_supervisor_bind(
13927 &session_id,
13928 supervisor.name.to_string(),
13929 supervisor.peer_id.as_str(),
13930 supervisor.address.to_string(),
13931 signing_public_key_for_descriptor(&supervisor),
13932 1,
13933 )
13934 .await
13935 .expect("pre-bind supervisor");
13936 adapter
13937 .test_install_session_peer_comms_handle_on_runtime(&session_id, runtime.as_ref())
13938 .await
13939 .expect("install adapter session peer-comms handle");
13940
13941 let current = BridgeMemberIncarnation {
13942 mob_id: "mob".to_string(),
13943 agent_identity: "worker".to_string(),
13944 host_id: "host".to_string(),
13945 binding_generation: 2,
13946 member_session_id: session_id.to_string(),
13947 generation: 1,
13948 fence_token: 7,
13949 };
13950 register_member_incarnation(&adapter, session_id.clone(), current.clone())
13951 .await
13952 .expect("register current member incarnation");
13953 let mut delayed = current;
13954 delayed.binding_generation = 1;
13955 let command = BridgeCommand::DeclareMemberOutboundTaint(BridgeOutboundTaintPayload {
13956 supervisor: supervisor.clone().into(),
13957 epoch: 1,
13958 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
13959 target: Some(BridgeOutboundTaintTarget::Placed(delayed)),
13960 taint: Some(meerkat_core::comms::SenderContentTaint::Tainted),
13961 });
13962 let interaction_id = InteractionId(Uuid::new_v4());
13963 let ingress = PeerIngressFact::peer(
13964 interaction_id,
13965 PeerInputClass::ActionableRequest,
13966 meerkat_core::PeerIngressKind::Request,
13967 Some(meerkat_core::PeerIngressAuthDecision::Required),
13968 PeerIngressIdentity::new(
13969 supervisor.peer_id,
13970 supervisor.name.as_str(),
13971 meerkat_core::PeerIngressConvention::Request {
13972 request_id: interaction_id.to_string(),
13973 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
13974 },
13975 )
13976 .with_signing_pubkey(supervisor.pubkey),
13977 );
13978 let candidate = bridge_candidate_with_ingress(supervisor.name.as_str(), &command, ingress);
13979 assert!(
13980 try_handle_supervisor_bridge_command(
13981 &adapter,
13982 &session_id,
13983 &(runtime.clone() as Arc<dyn CommsRuntime>),
13984 &candidate,
13985 )
13986 .await
13987 );
13988 assert_eq!(
13989 runtime.outbound_content_taint(),
13990 None,
13991 "a delayed G1 declaration must be rejected before mutating the revived G2 runtime"
13992 );
13993
13994 let legacy_command =
13995 BridgeCommand::DeclareMemberOutboundTaint(BridgeOutboundTaintPayload {
13996 supervisor: supervisor.clone().into(),
13997 epoch: 1,
13998 protocol_version: BridgeProtocolVersion::V3,
13999 target: None,
14000 taint: Some(meerkat_core::comms::SenderContentTaint::Tainted),
14001 });
14002 let legacy_id = InteractionId(Uuid::new_v4());
14003 let legacy_ingress = PeerIngressFact::peer(
14004 legacy_id,
14005 PeerInputClass::ActionableRequest,
14006 meerkat_core::PeerIngressKind::Request,
14007 Some(meerkat_core::PeerIngressAuthDecision::Required),
14008 PeerIngressIdentity::new(
14009 supervisor.peer_id,
14010 supervisor.name.as_str(),
14011 meerkat_core::PeerIngressConvention::Request {
14012 request_id: legacy_id.to_string(),
14013 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
14014 },
14015 )
14016 .with_signing_pubkey(supervisor.pubkey),
14017 );
14018 let legacy_candidate = bridge_candidate_with_ingress(
14019 supervisor.name.as_str(),
14020 &legacy_command,
14021 legacy_ingress,
14022 );
14023 assert!(
14024 try_handle_supervisor_bridge_command(
14025 &adapter,
14026 &session_id,
14027 &(runtime.clone() as Arc<dyn CommsRuntime>),
14028 &legacy_candidate,
14029 )
14030 .await
14031 );
14032 assert_eq!(
14033 runtime.outbound_content_taint(),
14034 None,
14035 "legacy omitted-target declarations must not address any registered placed incarnation"
14036 );
14037 }
14038
14039 #[test]
14040 fn delayed_g1_peer_message_fence_does_not_match_revived_g2_recipient() {
14041 let current = BridgeMemberIncarnation {
14042 mob_id: "mob".to_string(),
14043 agent_identity: "worker".to_string(),
14044 host_id: "host".to_string(),
14045 binding_generation: 2,
14046 member_session_id: "session".to_string(),
14047 generation: 1,
14048 fence_token: 7,
14049 };
14050 let mut delayed = meerkat_core::comms::PeerRecipientIncarnation {
14051 mob_id: current.mob_id.clone(),
14052 agent_identity: current.agent_identity.clone(),
14053 host_id: current.host_id.clone(),
14054 binding_generation: 1,
14055 member_session_id: current.member_session_id.clone(),
14056 generation: current.generation,
14057 fence_token: current.fence_token,
14058 };
14059 assert!(
14060 !peer_recipient_incarnation_matches(&delayed, ¤t),
14061 "a signed G1 envelope delayed across revoke/rebind cannot address G2"
14062 );
14063 delayed.binding_generation = 2;
14064 assert!(peer_recipient_incarnation_matches(&delayed, ¤t));
14065 }
14066
14067 #[tokio::test]
14068 async fn declare_member_outbound_taint_from_unauthorized_sender_installs_nothing() {
14069 let runtime = Arc::new(
14070 meerkat_comms::CommsRuntime::inproc_only("receiver-taint-unauthorized").unwrap(),
14071 );
14072 let supervisor_runtime = Arc::new(
14073 meerkat_comms::CommsRuntime::inproc_only("mob/__mob_supervisor_taint_real__").unwrap(),
14074 );
14075 let adapter = Arc::new(MeerkatMachine::ephemeral());
14076 let session_id = SessionId::new();
14077 adapter
14078 .register_session(session_id.clone())
14079 .await
14080 .expect("register session");
14081 let supervisor =
14082 trusted_peer_from_runtime("mob/__mob_supervisor_taint_real__", &supervisor_runtime);
14083 add_test_projection_trust(runtime.as_ref(), supervisor.clone(), "trust supervisor").await;
14084 adapter
14085 .stage_supervisor_bind(
14086 &session_id,
14087 supervisor.name.to_string(),
14088 supervisor.peer_id.as_str(),
14089 supervisor.address.to_string(),
14090 signing_public_key_for_descriptor(&supervisor),
14091 1,
14092 )
14093 .await
14094 .expect("pre-bind supervisor");
14095 let impostor_peer_id = PeerId::new();
14099 let candidate = bridge_candidate(
14100 &impostor_peer_id.as_str(),
14101 &BridgeCommand::DeclareMemberOutboundTaint(BridgeOutboundTaintPayload {
14102 supervisor: supervisor.clone().into(),
14103 epoch: 1,
14104 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
14105 target: Some(BridgeOutboundTaintTarget::PeerOnly),
14106 taint: Some(meerkat_core::comms::SenderContentTaint::Tainted),
14107 }),
14108 );
14109
14110 assert!(
14111 try_handle_supervisor_bridge_command(
14112 &adapter,
14113 &session_id,
14114 &(runtime.clone() as Arc<dyn CommsRuntime>),
14115 &candidate,
14116 )
14117 .await,
14118 "unauthorized declare must still be handled (and rejected)"
14119 );
14120 assert_eq!(
14121 runtime.outbound_content_taint(),
14122 None,
14123 "an unauthorized sender must not install an outbound taint declaration"
14124 );
14125 }
14126
14127 #[tokio::test]
14128 async fn wire_member_rejects_mob_overlay_for_different_recipient() {
14129 let runtime =
14130 Arc::new(meerkat_comms::CommsRuntime::inproc_only("receiver-wire-mismatch").unwrap());
14131 let supervisor_runtime = Arc::new(
14132 meerkat_comms::CommsRuntime::inproc_only("mob/__mob_supervisor_mismatch__").unwrap(),
14133 );
14134 let peer_runtime =
14135 Arc::new(meerkat_comms::CommsRuntime::inproc_only("peer-wire-mismatch").unwrap());
14136 let adapter = Arc::new(MeerkatMachine::ephemeral());
14137 let session_id = SessionId::new();
14138 adapter
14139 .register_session(session_id.clone())
14140 .await
14141 .expect("register session");
14142 let supervisor =
14143 trusted_peer_from_runtime("mob/__mob_supervisor_mismatch__", &supervisor_runtime);
14144 add_test_projection_trust(runtime.as_ref(), supervisor.clone(), "trust supervisor").await;
14145 adapter
14146 .stage_supervisor_bind(
14147 &session_id,
14148 supervisor.name.to_string(),
14149 supervisor.peer_id.as_str(),
14150 supervisor.address.to_string(),
14151 signing_public_key_for_descriptor(&supervisor),
14152 1,
14153 )
14154 .await
14155 .expect("pre-bind supervisor");
14156 let peer_spec = trusted_peer_from_runtime("peer-wire-mismatch", &peer_runtime);
14157 let candidate = bridge_candidate(
14158 &supervisor.peer_id.as_str(),
14159 &BridgeCommand::WireMember(BridgePeerWiringPayload {
14160 supervisor: supervisor.clone().into(),
14161 epoch: 1,
14162 protocol_version: SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
14163 peer_spec: peer_spec.clone().into(),
14164 mob_peer_overlay: Some(BridgeMobPeerOverlayHandoff {
14165 recipient_peer_id: PeerId::new().as_str(),
14166 topology_epoch: 1,
14167 peer_specs: vec![peer_spec.clone().into()],
14168 }),
14169 }),
14170 );
14171
14172 assert!(
14173 try_handle_supervisor_bridge_command(
14174 &adapter,
14175 &session_id,
14176 &(runtime.clone() as Arc<dyn CommsRuntime>),
14177 &candidate,
14178 )
14179 .await,
14180 "wire bridge command should be handled"
14181 );
14182 let peers = runtime.peers().await;
14183 assert!(
14184 peers.iter().all(|entry| entry.peer_id != peer_spec.peer_id),
14185 "MobMachine overlay handoff for another recipient must not update local peer trust"
14186 );
14187 }
14188
14189 enum TestWaiterEntry {
14192 Waiting(tokio::sync::oneshot::Sender<PeerInputCandidate>),
14193 TimedOut,
14194 }
14195
14196 struct WaiterReplyRuntime {
14200 notify: Arc<tokio::sync::Notify>,
14201 candidate: std::sync::Mutex<Option<PeerInputCandidate>>,
14202 peer_handle: Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>>,
14203 completed_count: std::sync::atomic::AtomicUsize,
14204 waiters: std::sync::Mutex<HashMap<Uuid, TestWaiterEntry>>,
14205 }
14206
14207 impl WaiterReplyRuntime {
14208 fn new(
14209 candidate: PeerInputCandidate,
14210 peer_handle: Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>>,
14211 in_reply_to: InteractionId,
14212 entry: TestWaiterEntry,
14213 ) -> Self {
14214 let mut waiters = HashMap::new();
14215 waiters.insert(in_reply_to.0, entry);
14216 Self {
14217 notify: Arc::new(tokio::sync::Notify::new()),
14218 candidate: std::sync::Mutex::new(Some(candidate)),
14219 peer_handle,
14220 completed_count: std::sync::atomic::AtomicUsize::new(0),
14221 waiters: std::sync::Mutex::new(waiters),
14222 }
14223 }
14224
14225 fn completed_count(&self) -> usize {
14226 self.completed_count
14227 .load(std::sync::atomic::Ordering::SeqCst)
14228 }
14229
14230 fn waiter_entry_count(&self) -> usize {
14231 self.waiters.lock().expect("waiters mutex").len()
14232 }
14233 }
14234
14235 #[async_trait::async_trait]
14236 impl CommsRuntime for WaiterReplyRuntime {
14237 async fn drain_messages(&self) -> Vec<String> {
14238 Vec::new()
14239 }
14240
14241 fn inbox_notify(&self) -> Arc<tokio::sync::Notify> {
14242 self.notify.clone()
14243 }
14244
14245 fn peer_interaction_handle(
14246 &self,
14247 ) -> Option<Arc<dyn meerkat_core::handles::PeerInteractionHandle>> {
14248 self.peer_handle.clone()
14249 }
14250
14251 async fn drain_classified_inbox_interactions(
14252 &self,
14253 ) -> Result<
14254 Vec<meerkat_core::interaction::ClassifiedInboxInteraction>,
14255 meerkat_core::agent::CommsCapabilityError,
14256 > {
14257 Ok(self
14258 .candidate
14259 .lock()
14260 .expect("candidate mutex")
14261 .take()
14262 .into_iter()
14263 .collect())
14264 }
14265
14266 fn mark_interaction_complete(&self, _id: &InteractionId) {
14267 self.completed_count
14268 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
14269 }
14270
14271 fn take_bridge_reply_waiter(
14272 &self,
14273 in_reply_to: &InteractionId,
14274 ) -> Option<tokio::sync::oneshot::Sender<PeerInputCandidate>> {
14275 match self
14276 .waiters
14277 .lock()
14278 .expect("waiters mutex")
14279 .remove(&in_reply_to.0)
14280 {
14281 Some(TestWaiterEntry::Waiting(sender)) => Some(sender),
14282 Some(TestWaiterEntry::TimedOut) | None => None,
14283 }
14284 }
14285
14286 fn has_bridge_reply_waiter(&self, in_reply_to: &InteractionId) -> bool {
14287 self.waiters
14288 .lock()
14289 .expect("waiters mutex")
14290 .contains_key(&in_reply_to.0)
14291 }
14292 }
14293
14294 fn response_candidate_replying_to(
14295 class: PeerInputClass,
14296 response_terminality: Option<meerkat_core::interaction::TerminalityClass>,
14297 in_reply_to: InteractionId,
14298 ) -> PeerInputCandidate {
14299 let mut candidate = response_candidate(class, response_terminality);
14300 if let InteractionContent::Response {
14301 in_reply_to: reply_slot,
14302 ..
14303 } = &mut candidate.interaction.content
14304 {
14305 *reply_slot = in_reply_to;
14306 }
14307 candidate
14308 }
14309
14310 #[tokio::test]
14315 async fn terminal_response_with_registered_waiter_is_delivered_not_injected() {
14316 let adapter = Arc::new(MeerkatMachine::ephemeral());
14317 let session_id = SessionId::new();
14318 adapter
14319 .register_session(session_id.clone())
14320 .await
14321 .expect("register session");
14322
14323 let in_reply_to = InteractionId(Uuid::new_v4());
14324 let candidate = response_candidate_replying_to(
14325 PeerInputClass::ResponseTerminal,
14326 Some(meerkat_core::interaction::TerminalityClass::Terminal {
14327 disposition: meerkat_core::interaction::TerminalDisposition::Completed,
14328 }),
14329 in_reply_to,
14330 );
14331 let (waiter_tx, waiter_rx) = tokio::sync::oneshot::channel();
14332 let runtime = Arc::new(WaiterReplyRuntime::new(
14333 candidate,
14334 None,
14335 in_reply_to,
14336 TestWaiterEntry::Waiting(waiter_tx),
14337 ));
14338
14339 let drain = spawn_authorized_test_comms_drain(
14340 adapter.clone(),
14341 session_id.clone(),
14342 runtime.clone(),
14343 Duration::from_millis(10),
14344 )
14345 .await;
14346 tokio::time::timeout(Duration::from_secs(1), drain)
14347 .await
14348 .expect("drain should exit after one candidate")
14349 .expect("drain task should not panic");
14350
14351 let delivered = waiter_rx.await.expect("waiter must receive the candidate");
14352 assert!(
14353 matches!(
14354 delivered.response_terminality,
14355 Some(meerkat_core::interaction::TerminalityClass::Terminal {
14356 disposition: meerkat_core::interaction::TerminalDisposition::Completed,
14357 })
14358 ),
14359 "typed terminality must reach the waiter intact"
14360 );
14361 assert!(
14362 matches!(
14363 &delivered.interaction.content,
14364 InteractionContent::Response { in_reply_to: got, .. } if *got == in_reply_to
14365 ),
14366 "the waiter must receive the correlated Response"
14367 );
14368 assert!(
14369 runtime.completed_count() >= 1,
14370 "waiter delivery must mark the interaction complete"
14371 );
14372 assert_eq!(
14373 runtime.waiter_entry_count(),
14374 0,
14375 "take must consume the waiter"
14376 );
14377 let snapshot = adapter
14378 .meerkat_machine_spine_snapshot(&session_id)
14379 .await
14380 .expect("registered session snapshot");
14381 assert_eq!(
14382 snapshot.ledger.input_count, 0,
14383 "a waiter-consumed reply must never become session input"
14384 );
14385 }
14386
14387 #[tokio::test]
14391 async fn tombstoned_waiter_consumes_late_terminal_reply_without_injection() {
14392 let adapter = Arc::new(MeerkatMachine::ephemeral());
14393 let session_id = SessionId::new();
14394 adapter
14395 .register_session(session_id.clone())
14396 .await
14397 .expect("register session");
14398
14399 let in_reply_to = InteractionId(Uuid::new_v4());
14400 let candidate = response_candidate_replying_to(
14401 PeerInputClass::ResponseTerminal,
14402 Some(meerkat_core::interaction::TerminalityClass::Terminal {
14403 disposition: meerkat_core::interaction::TerminalDisposition::Completed,
14404 }),
14405 in_reply_to,
14406 );
14407 let runtime = Arc::new(WaiterReplyRuntime::new(
14408 candidate,
14409 None,
14410 in_reply_to,
14411 TestWaiterEntry::TimedOut,
14412 ));
14413
14414 let drain = spawn_authorized_test_comms_drain(
14415 adapter.clone(),
14416 session_id.clone(),
14417 runtime.clone(),
14418 Duration::from_millis(10),
14419 )
14420 .await;
14421 tokio::time::timeout(Duration::from_secs(1), drain)
14422 .await
14423 .expect("drain should exit after one candidate")
14424 .expect("drain task should not panic");
14425
14426 assert_eq!(
14427 runtime.waiter_entry_count(),
14428 0,
14429 "the tombstone must be consumed by the late reply"
14430 );
14431 assert!(
14432 runtime.completed_count() >= 1,
14433 "late-reply discard must mark the interaction complete"
14434 );
14435 let snapshot = adapter
14436 .meerkat_machine_spine_snapshot(&session_id)
14437 .await
14438 .expect("registered session snapshot");
14439 assert_eq!(
14440 snapshot.ledger.input_count, 0,
14441 "a discarded late reply must never leak into the session"
14442 );
14443 }
14444
14445 #[tokio::test]
14449 async fn progress_response_with_registered_waiter_is_suppressed() {
14450 let adapter = Arc::new(MeerkatMachine::ephemeral());
14451 let session_id = SessionId::new();
14452 adapter
14453 .register_session(session_id.clone())
14454 .await
14455 .expect("register session");
14456
14457 let in_reply_to = InteractionId(Uuid::new_v4());
14458 let candidate = response_candidate_replying_to(
14459 PeerInputClass::ResponseProgress,
14460 Some(meerkat_core::interaction::TerminalityClass::Progress),
14461 in_reply_to,
14462 );
14463 let (waiter_tx, _waiter_rx) = tokio::sync::oneshot::channel();
14464 let runtime = Arc::new(WaiterReplyRuntime::new(
14465 candidate,
14466 None,
14467 in_reply_to,
14468 TestWaiterEntry::Waiting(waiter_tx),
14469 ));
14470
14471 let drain = spawn_authorized_test_comms_drain(
14472 adapter.clone(),
14473 session_id.clone(),
14474 runtime.clone(),
14475 Duration::from_millis(10),
14476 )
14477 .await;
14478 tokio::time::timeout(Duration::from_secs(1), drain)
14479 .await
14480 .expect("drain should exit after one candidate")
14481 .expect("drain task should not panic");
14482
14483 assert_eq!(
14484 runtime.waiter_entry_count(),
14485 1,
14486 "a Progress reply must not consume the waiter"
14487 );
14488 let snapshot = adapter
14489 .meerkat_machine_spine_snapshot(&session_id)
14490 .await
14491 .expect("registered session snapshot");
14492 assert_eq!(
14493 snapshot.ledger.input_count, 0,
14494 "a Progress reply for a waited interaction must not become session input"
14495 );
14496 }
14497
14498 #[tokio::test]
14502 async fn terminal_response_without_waiter_keeps_existing_injection_path() {
14503 let adapter = Arc::new(MeerkatMachine::ephemeral());
14504 let session_id = SessionId::new();
14505 adapter
14506 .register_session(session_id.clone())
14507 .await
14508 .expect("register session");
14509
14510 let runtime = Arc::new(OneShotPeerRequestRuntime::new(
14511 response_candidate(
14512 PeerInputClass::ResponseTerminal,
14513 Some(meerkat_core::interaction::TerminalityClass::Terminal {
14514 disposition: meerkat_core::interaction::TerminalDisposition::Completed,
14515 }),
14516 ),
14517 None,
14518 ));
14519 let drain = spawn_authorized_test_comms_drain(
14520 adapter.clone(),
14521 session_id.clone(),
14522 runtime.clone(),
14523 Duration::from_millis(10),
14524 )
14525 .await;
14526 tokio::time::timeout(Duration::from_secs(1), drain)
14527 .await
14528 .expect("drain should exit after one candidate")
14529 .expect("drain task should not panic");
14530
14531 let snapshot = adapter
14532 .meerkat_machine_spine_snapshot(&session_id)
14533 .await
14534 .expect("registered session snapshot");
14535 assert_eq!(
14536 snapshot.ledger.input_count, 1,
14537 "without a waiter the terminal response must still inject as session input"
14538 );
14539 }
14540
14541 #[tokio::test]
14548 async fn bind_member_invalid_bootstrap_never_stages_payload_reply_address() {
14549 let runtime_impl = bootstrap_runtime(
14550 PEER_ID_RECEIVER,
14551 "inproc://receiver",
14552 Some("expected-token"),
14553 );
14554 let staged = runtime_impl.staged_reply_endpoints.clone();
14555 let staged_correlated = runtime_impl.staged_correlated_reply_endpoints.clone();
14556 let sent = runtime_impl.sent_commands.clone();
14557 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
14558 let adapter = Arc::new(MeerkatMachine::ephemeral());
14559 let session_id = SessionId::new();
14560 adapter
14561 .register_session(session_id.clone())
14562 .await
14563 .expect("register session");
14564
14565 for payload_address in [
14566 "tcp://127.0.0.1:9",
14567 "uds:///tmp/meerkat-payload-selected-reply.sock",
14568 ] {
14569 let mut payload = sample_bind_payload();
14570 payload.bootstrap_token = "wrong-token".into();
14571 payload.supervisor.address = payload_address.to_string();
14572 let sender = pubkey_sender_for_bridge_spec(&payload.supervisor);
14575 let candidate = bridge_candidate(&sender, &BridgeCommand::BindMember(payload));
14576
14577 assert!(
14578 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate,)
14579 .await
14580 );
14581 }
14582
14583 let mut open_payload = sample_bind_payload();
14587 open_payload.bootstrap_token = "wrong-token".into();
14588 open_payload.supervisor.address = "tcp://127.0.0.1:19".to_string();
14589 let open_sender = open_payload.supervisor.name.clone();
14590 let open_sender_peer_id =
14591 meerkat_comms::PubKey::new(open_payload.supervisor.pubkey).to_peer_id();
14592 let open_ingress =
14593 bridge_sender_fact_with_display(open_sender_peer_id, open_sender.as_str());
14594 let open_candidate = bridge_candidate_with_ingress(
14595 open_sender.as_str(),
14596 &BridgeCommand::BindMember(open_payload),
14597 open_ingress,
14598 );
14599 assert!(
14600 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &open_candidate,)
14601 .await
14602 );
14603
14604 let staged = staged.lock().expect("staged reply endpoints mutex").clone();
14605 assert!(
14606 staged.is_empty(),
14607 "payload-selected TCP/UDS addresses must never become callback routes: {staged:?}"
14608 );
14609 let staged_correlated = staged_correlated
14610 .lock()
14611 .expect("staged correlated reply endpoints mutex")
14612 .clone();
14613 assert!(
14614 staged_correlated.is_empty(),
14615 "payload-selected TCP/UDS/open-auth addresses must never become correlated callback routes: {staged_correlated:?}"
14616 );
14617 let replies = recorded_bridge_replies(&sent).await;
14618 assert_eq!(
14619 replies.len(),
14620 3,
14621 "one rejection attempt per malicious payload"
14622 );
14623 for (status, reply) in replies {
14624 assert_eq!(status, meerkat_core::interaction::ResponseStatus::Failed);
14625 assert!(
14626 matches!(&reply, BridgeReply::Rejected { .. }),
14627 "invalid bootstrap must produce a typed rejection attempt, got {reply:?}"
14628 );
14629 }
14630 }
14631
14632 #[tokio::test]
14638 async fn invalid_bootstrap_stages_exact_source_confined_callback_tuple() {
14639 let runtime_impl = bootstrap_runtime(
14640 PEER_ID_RECEIVER,
14641 "inproc://receiver",
14642 Some("expected-token"),
14643 );
14644 let staged_legacy = runtime_impl.staged_reply_endpoints.clone();
14645 let staged_correlated = runtime_impl.staged_correlated_reply_endpoints.clone();
14646 let unstaged_correlated = runtime_impl.unstaged_correlated_reply_endpoints.clone();
14647 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
14648 let adapter = Arc::new(MeerkatMachine::ephemeral());
14649 let session_id = SessionId::new();
14650 adapter
14651 .register_session(session_id.clone())
14652 .await
14653 .expect("register session");
14654
14655 let mut payload = sample_bind_payload();
14656 payload.bootstrap_token = "wrong-token".into();
14657 payload.supervisor.address =
14658 "uds:///tmp/payload-address-is-not-callback-authority.sock".to_string();
14659 let sender_name = payload.supervisor.name.clone();
14660 let signer_pubkey = payload.supervisor.pubkey;
14661 let sender_peer_id = meerkat_comms::PubKey::new(signer_pubkey).to_peer_id();
14662 let interaction_id = InteractionId(Uuid::new_v4());
14663 let callback = PeerAddress::parse("tcp://192.0.2.44:4311")
14664 .expect("source-confined callback endpoint parses");
14665 let ingress = PeerIngressFact::peer(
14666 interaction_id,
14667 PeerInputClass::ActionableRequest,
14668 meerkat_core::PeerIngressKind::Request,
14669 Some(meerkat_core::PeerIngressAuthDecision::Exempt(
14670 meerkat_core::PeerIngressAuthExemption::SupervisorBridge,
14671 )),
14672 PeerIngressIdentity::new(
14673 sender_peer_id,
14674 sender_name.as_str(),
14675 meerkat_core::PeerIngressConvention::Request {
14676 request_id: interaction_id.to_string(),
14677 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
14678 },
14679 )
14680 .with_signing_pubkey(signer_pubkey),
14681 )
14682 .with_declared_reply_endpoint(Some(callback.clone()));
14683 let candidate = bridge_candidate_with_ingress(
14684 sender_name.as_str(),
14685 &BridgeCommand::BindMember(payload),
14686 ingress,
14687 );
14688
14689 assert!(
14690 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate,)
14691 .await
14692 );
14693 assert!(
14694 staged_legacy
14695 .lock()
14696 .expect("staged legacy reply endpoints mutex")
14697 .is_empty(),
14698 "a correlated ingress callback must never use the legacy staging seam"
14699 );
14700 assert_eq!(
14701 staged_correlated
14702 .lock()
14703 .expect("staged correlated reply endpoints mutex")
14704 .as_slice(),
14705 &[(sender_peer_id, interaction_id, signer_pubkey, callback)],
14706 "bridge response staging must preserve peer, correlation, signer, and source-confined endpoint"
14707 );
14708 assert!(
14709 unstaged_correlated
14710 .lock()
14711 .expect("unstaged correlated reply endpoints mutex")
14712 .is_empty(),
14713 "a successfully recorded response must leave cleanup to the one-shot Router send"
14714 );
14715 }
14716
14717 #[tokio::test]
14720 async fn undecodable_bridge_command_stages_no_reply_endpoint() {
14721 let runtime_impl = bootstrap_runtime(
14722 PEER_ID_RECEIVER,
14723 "inproc://receiver",
14724 Some("expected-token"),
14725 );
14726 let staged = runtime_impl.staged_reply_endpoints.clone();
14727 let sent = runtime_impl.sent_commands.clone();
14728 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
14729 let adapter = Arc::new(MeerkatMachine::ephemeral());
14730 let session_id = SessionId::new();
14731 adapter
14732 .register_session(session_id.clone())
14733 .await
14734 .expect("register session");
14735
14736 let id = InteractionId(Uuid::new_v4());
14737 let sender_key = meerkat_comms::PubKey::new([0xbb; 32]).to_pubkey_string();
14738 let ingress = bridge_sender_fact_with_id(id, &sender_key);
14739 let candidate = PeerInputCandidate {
14740 interaction: InboxInteraction {
14741 sender_taint: None,
14742 id,
14743 from_route: None,
14744 from: sender_key.clone(),
14745 content: InteractionContent::Request {
14746 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
14747 params: json!({ "not": "a bridge command" }),
14748 blocks: None,
14749 },
14750 rendered_text: String::new(),
14751 handling_mode: HandlingMode::Queue,
14752 render_metadata: None,
14753 objective_id: None,
14754 },
14755 ingress,
14756 lifecycle_peer: None,
14757 response_terminality: None,
14758 };
14759
14760 assert!(
14761 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate).await
14762 );
14763 assert!(
14764 staged
14765 .lock()
14766 .expect("staged reply endpoints mutex")
14767 .is_empty(),
14768 "pre-decode failures must stage nothing"
14769 );
14770 let replies = recorded_bridge_replies(&sent).await;
14771 assert_eq!(replies.len(), 1, "exactly one rejection reply");
14772 assert!(matches!(&replies[0].1, BridgeReply::Rejected { .. }));
14773 }
14774
14775 #[tokio::test]
14778 async fn bind_member_accept_path_stages_no_reply_endpoint() {
14779 let payload = sample_bind_payload();
14780 let sender = pubkey_sender_for_bridge_spec(&payload.supervisor);
14781 let runtime_impl = bootstrap_runtime(
14782 PEER_ID_RECEIVER,
14783 "inproc://receiver",
14784 Some("expected-token"),
14785 );
14786 let staged = runtime_impl.staged_reply_endpoints.clone();
14787 let sent = runtime_impl.sent_commands.clone();
14788 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
14789 let adapter = Arc::new(MeerkatMachine::ephemeral());
14790 let session_id = SessionId::new();
14791 adapter
14792 .register_session(session_id.clone())
14793 .await
14794 .expect("register session");
14795 let candidate = bridge_candidate(&sender, &BridgeCommand::BindMember(payload));
14796
14797 assert!(
14798 try_handle_supervisor_bridge_command(&adapter, &session_id, &runtime, &candidate).await
14799 );
14800 let replies = recorded_bridge_replies(&sent).await;
14801 assert_eq!(replies.len(), 1);
14802 assert!(
14803 matches!(&replies[0].1, BridgeReply::BindMember(_)),
14804 "expected a successful bind reply, got {:?}",
14805 replies[0].1
14806 );
14807 assert!(
14808 staged
14809 .lock()
14810 .expect("staged reply endpoints mutex")
14811 .is_empty(),
14812 "the accept path must not stage a declared reply endpoint"
14813 );
14814 }
14815
14816 #[tokio::test]
14822 async fn bind_supervisor_for_materialized_session_installs_binding_and_private_trust() {
14823 let runtime_impl = bootstrap_runtime(
14824 PEER_ID_RECEIVER,
14825 "inproc://receiver",
14826 Some("expected-token"),
14827 );
14828 let trusted_peer_ids = runtime_impl.trusted_peer_ids.clone();
14829 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
14830 let adapter = Arc::new(MeerkatMachine::ephemeral());
14831 let session_id = SessionId::new();
14832 adapter
14833 .register_session(session_id.clone())
14834 .await
14835 .expect("register session");
14836 let supervisor = trusted_supervisor_descriptor(0xbb);
14837
14838 bind_supervisor_for_materialized_session(
14839 &adapter,
14840 &session_id,
14841 runtime.as_ref(),
14842 &supervisor,
14843 1,
14844 )
14845 .await
14846 .expect("host-seeded supervisor bind must succeed");
14847
14848 assert!(
14849 !matches!(
14850 adapter.supervisor_binding(&session_id).await,
14851 SupervisorBinding::Unbound
14852 ),
14853 "the supervisor binding must be installed"
14854 );
14855 assert!(
14856 trusted_peer_ids
14857 .lock()
14858 .await
14859 .contains(&supervisor.peer_id.to_string()),
14860 "the private supervisor admission edge must be installed"
14861 );
14862
14863 bind_supervisor_for_materialized_session(
14866 &adapter,
14867 &session_id,
14868 runtime.as_ref(),
14869 &supervisor,
14870 1,
14871 )
14872 .await
14873 .expect("idempotent host-seeded rebind must succeed");
14874 assert!(
14875 !matches!(
14876 adapter.supervisor_binding(&session_id).await,
14877 SupervisorBinding::Unbound
14878 ),
14879 "the binding must survive the idempotent rebind"
14880 );
14881 }
14882
14883 #[tokio::test]
14887 async fn authorize_supervisor_for_materialized_session_advances_same_peer_and_key() {
14888 let runtime_impl = bootstrap_runtime(
14889 PEER_ID_RECEIVER,
14890 "inproc://receiver",
14891 Some("expected-token"),
14892 );
14893 let trusted_peer_ids = runtime_impl.trusted_peer_ids.clone();
14894 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
14895 let adapter = Arc::new(MeerkatMachine::ephemeral());
14896 let session_id = SessionId::new();
14897 adapter
14898 .register_session(session_id.clone())
14899 .await
14900 .expect("register session");
14901 let original = trusted_supervisor_descriptor(0xbb);
14902 let original_peer_id = original.peer_id.to_string();
14903 bind_supervisor_for_materialized_session(
14904 &adapter,
14905 &session_id,
14906 runtime.as_ref(),
14907 &original,
14908 7,
14909 )
14910 .await
14911 .expect("initial host-seeded supervisor bind");
14912 let refreshed_address = "inproc://mob/__mob_supervisor__/restarted";
14913 let refreshed = TrustedPeerDescriptor::unsigned_with_pubkey(
14914 original.name.clone(),
14915 original.peer_id.to_string(),
14916 original.pubkey,
14917 refreshed_address,
14918 )
14919 .expect("same-authority refreshed descriptor");
14920
14921 authorize_supervisor_for_materialized_session(
14922 &adapter,
14923 &session_id,
14924 runtime.as_ref(),
14925 &refreshed,
14926 8,
14927 )
14928 .await
14929 .expect("same-peer/key monotonic host refresh succeeds");
14930
14931 assert!(matches!(
14932 adapter.supervisor_binding(&session_id).await,
14933 SupervisorBinding::Bound {
14934 ref peer_id,
14935 ref address,
14936 epoch: 8,
14937 ..
14938 } if peer_id == &original_peer_id && address == refreshed_address
14939 ));
14940 assert!(
14941 trusted_peer_ids.lock().await.contains(&original_peer_id),
14942 "the refreshed private supervisor trust edge is published before success"
14943 );
14944 }
14945
14946 #[tokio::test]
14951 async fn authorize_supervisor_for_materialized_session_publication_failure_is_fail_closed_and_retryable()
14952 {
14953 let original = trusted_supervisor_descriptor(0xbb);
14954 let original_peer_id = original.peer_id.to_string();
14955 let refreshed_address = "inproc://mob/__mob_supervisor__/restarted";
14956 let refreshed = TrustedPeerDescriptor::unsigned_with_pubkey(
14957 original.name.clone(),
14958 original.peer_id.to_string(),
14959 original.pubkey,
14960 refreshed_address,
14961 )
14962 .expect("same-authority refreshed descriptor");
14963
14964 let mut failing_runtime_impl = bootstrap_runtime(
14965 PEER_ID_RECEIVER,
14966 "inproc://receiver",
14967 Some("expected-token"),
14968 );
14969 failing_runtime_impl
14970 .add_trusted_peer_errors
14971 .insert(original_peer_id.clone(), "boom".to_string());
14972 let failing_trusted_peer_ids = failing_runtime_impl.trusted_peer_ids.clone();
14973 let failing_runtime: Arc<dyn CommsRuntime> = Arc::new(failing_runtime_impl);
14974 let adapter = Arc::new(MeerkatMachine::ephemeral());
14975 let session_id = SessionId::new();
14976 adapter
14977 .register_session(session_id.clone())
14978 .await
14979 .expect("register session");
14980 adapter
14981 .stage_local_endpoint_for_comms_runtime(&session_id, failing_runtime.as_ref())
14982 .await
14983 .expect("stage local endpoint");
14984 adapter
14985 .stage_supervisor_bind(
14986 &session_id,
14987 original.name.to_string(),
14988 original.peer_id.as_str(),
14989 original.address.to_string(),
14990 signing_public_key_for_descriptor(&original),
14991 7,
14992 )
14993 .await
14994 .expect("seed current binding");
14995 adapter
14996 .stage_supervisor_trust_published(&session_id, original_peer_id.clone(), 7)
14997 .await
14998 .expect("close seeded publication");
14999 failing_trusted_peer_ids
15000 .lock()
15001 .await
15002 .insert(original_peer_id.clone());
15003
15004 let error = authorize_supervisor_for_materialized_session(
15005 &adapter,
15006 &session_id,
15007 failing_runtime.as_ref(),
15008 &refreshed,
15009 8,
15010 )
15011 .await
15012 .expect_err("synthetic trust publication failure must surface");
15013 assert!(matches!(
15014 error,
15015 SupervisorMaterializeAuthorizeError::Apply(_)
15016 ));
15017 assert!(matches!(
15018 adapter.supervisor_binding(&session_id).await,
15019 SupervisorBinding::Bound {
15020 ref peer_id,
15021 ref address,
15022 epoch: 8,
15023 ..
15024 } if peer_id == &original_peer_id && address == refreshed_address
15025 ));
15026 assert!(
15027 failing_trusted_peer_ids.lock().await.is_empty(),
15028 "failed replacement publication must remove the stale route"
15029 );
15030
15031 let retry_runtime_impl = bootstrap_runtime(
15032 PEER_ID_RECEIVER,
15033 "inproc://receiver",
15034 Some("expected-token"),
15035 );
15036 let retry_trusted_peer_ids = retry_runtime_impl.trusted_peer_ids.clone();
15037 let retry_runtime: Arc<dyn CommsRuntime> = Arc::new(retry_runtime_impl);
15038 authorize_supervisor_for_materialized_session(
15039 &adapter,
15040 &session_id,
15041 retry_runtime.as_ref(),
15042 &refreshed,
15043 8,
15044 )
15045 .await
15046 .expect("exact retry re-drives the pending private trust publication");
15047 assert!(
15048 retry_trusted_peer_ids
15049 .lock()
15050 .await
15051 .contains(&original_peer_id),
15052 "retry must publish the refreshed private route"
15053 );
15054 }
15055
15056 #[tokio::test]
15060 async fn bind_supervisor_for_materialized_session_rolls_back_on_publication_failure() {
15061 let supervisor = trusted_supervisor_descriptor(0xbb);
15062 let mut runtime_impl = bootstrap_runtime(
15063 PEER_ID_RECEIVER,
15064 "inproc://receiver",
15065 Some("expected-token"),
15066 );
15067 runtime_impl
15068 .add_trusted_peer_errors
15069 .insert(supervisor.peer_id.to_string(), "boom".to_string());
15070 let trusted_peer_ids = runtime_impl.trusted_peer_ids.clone();
15071 let runtime: Arc<dyn CommsRuntime> = Arc::new(runtime_impl);
15072 let adapter = Arc::new(MeerkatMachine::ephemeral());
15073 let session_id = SessionId::new();
15074 adapter
15075 .register_session(session_id.clone())
15076 .await
15077 .expect("register session");
15078
15079 let error = bind_supervisor_for_materialized_session(
15080 &adapter,
15081 &session_id,
15082 runtime.as_ref(),
15083 &supervisor,
15084 1,
15085 )
15086 .await
15087 .expect_err("trust publication failure must fail the bind");
15088 assert!(
15089 matches!(error, SupervisorMaterializeBindError::Commit(_)),
15090 "expected a Commit error, got {error:?}"
15091 );
15092 assert!(
15093 matches!(
15094 adapter.supervisor_binding(&session_id).await,
15095 SupervisorBinding::Unbound
15096 ),
15097 "a failed publication must roll the staged bind back to Unbound"
15098 );
15099 assert!(
15100 trusted_peer_ids.lock().await.is_empty(),
15101 "no trust edge may survive the rollback"
15102 );
15103 }
15104
15105 use crate::member_live::{MemberLiveError, MemberLiveHost, MemberLiveStatus};
15110 use meerkat_contracts::wire::supervisor_bridge::{
15111 BridgeLiveChannelPayload, BridgeLiveControlOutcome, BridgeLiveControlPayload,
15112 BridgeLiveControlVerb, BridgeLiveOpenPayload, BridgeLiveStatusPayload,
15113 };
15114 use meerkat_contracts::{
15115 LiveCloseStatus, LiveCommitInputStatus, LiveInterruptStatus, LiveOpenResult,
15116 LiveOpenTransport, LiveRefreshStatus, LiveTruncateStatus, RealtimeTurningMode,
15117 WireLiveAdapterStatus, WireLiveContinuityMode, WireLiveTransportBootstrap,
15118 };
15119
15120 #[test]
15125 fn member_bridge_capabilities_stay_live_free() {}
15126
15127 #[derive(Debug, Clone, PartialEq)]
15128 enum ScriptedLiveCall {
15129 Open {
15130 session: SessionId,
15131 turning_mode: Option<RealtimeTurningMode>,
15132 transport: Option<LiveOpenTransport>,
15133 },
15134 Close {
15135 session: SessionId,
15136 channel_id: String,
15137 },
15138 Status {
15139 session: SessionId,
15140 channel_id: Option<String>,
15141 },
15142 Control {
15143 session: SessionId,
15144 channel_id: String,
15145 verb: BridgeLiveControlVerb,
15146 },
15147 }
15148
15149 fn scripted_open_result() -> LiveOpenResult {
15150 LiveOpenResult {
15151 channel_id: "chan-scripted".to_string(),
15152 transport: WireLiveTransportBootstrap::Websocket {
15153 url: "wss://advertised.example:8443/live/ws?token=tok-1&channel=chan-scripted&format=pcm_24k_mono"
15154 .to_string(),
15155 token: "tok-1".to_string(),
15156 },
15157 capabilities: meerkat_core::live_adapter::LiveChannelCapabilities::default().into(),
15158 continuity: WireLiveContinuityMode::Fresh,
15159 }
15160 }
15161
15162 #[derive(Default)]
15166 struct ScriptedMemberLive {
15167 calls: std::sync::Mutex<Vec<ScriptedLiveCall>>,
15168 open_gate: Option<Arc<tokio::sync::Semaphore>>,
15169 }
15170
15171 impl ScriptedMemberLive {
15172 fn recorded(&self) -> Vec<ScriptedLiveCall> {
15173 self.calls.lock().expect("scripted live calls lock").clone()
15174 }
15175
15176 fn record(&self, call: ScriptedLiveCall) {
15177 self.calls
15178 .lock()
15179 .expect("scripted live calls lock")
15180 .push(call);
15181 }
15182 }
15183
15184 #[async_trait::async_trait]
15185 impl MemberLiveHost for ScriptedMemberLive {
15186 async fn open(
15187 &self,
15188 session: &SessionId,
15189 turning_mode: Option<RealtimeTurningMode>,
15190 transport: Option<LiveOpenTransport>,
15191 ) -> Result<LiveOpenResult, MemberLiveError> {
15192 self.record(ScriptedLiveCall::Open {
15193 session: session.clone(),
15194 turning_mode,
15195 transport,
15196 });
15197 if let Some(gate) = &self.open_gate {
15198 let _permit = gate.acquire().await.expect("open gate semaphore");
15199 }
15200 Ok(scripted_open_result())
15201 }
15202
15203 async fn close(
15204 &self,
15205 session: &SessionId,
15206 channel_id: &str,
15207 ) -> Result<LiveCloseStatus, MemberLiveError> {
15208 self.record(ScriptedLiveCall::Close {
15209 session: session.clone(),
15210 channel_id: channel_id.to_string(),
15211 });
15212 Ok(LiveCloseStatus::Closed)
15213 }
15214
15215 async fn status(
15216 &self,
15217 session: &SessionId,
15218 channel_id: Option<String>,
15219 ) -> Result<MemberLiveStatus, MemberLiveError> {
15220 self.record(ScriptedLiveCall::Status {
15221 session: session.clone(),
15222 channel_id: channel_id.clone(),
15223 });
15224 Ok(MemberLiveStatus {
15225 channel_id: channel_id.unwrap_or_else(|| "chan-scripted".to_string()),
15226 status: WireLiveAdapterStatus::Ready,
15227 })
15228 }
15229
15230 async fn control(
15231 &self,
15232 session: &SessionId,
15233 channel_id: &str,
15234 verb: BridgeLiveControlVerb,
15235 ) -> Result<BridgeLiveControlOutcome, MemberLiveError> {
15236 self.record(ScriptedLiveCall::Control {
15237 session: session.clone(),
15238 channel_id: channel_id.to_string(),
15239 verb: verb.clone(),
15240 });
15241 Ok(match verb {
15242 BridgeLiveControlVerb::CommitInput => BridgeLiveControlOutcome::CommitInput {
15243 status: LiveCommitInputStatus::Committed,
15244 },
15245 BridgeLiveControlVerb::Interrupt => BridgeLiveControlOutcome::Interrupt {
15246 status: LiveInterruptStatus::Interrupted,
15247 },
15248 BridgeLiveControlVerb::Truncate { .. } => BridgeLiveControlOutcome::Truncate {
15249 status: LiveTruncateStatus::Truncated,
15250 },
15251 BridgeLiveControlVerb::Refresh => BridgeLiveControlOutcome::Refresh {
15252 status: LiveRefreshStatus::Queued,
15253 },
15254 })
15255 }
15256 }
15257
15258 struct LiveArmFixture {
15259 adapter: Arc<MeerkatMachine>,
15260 session_id: SessionId,
15261 member_runtime: Arc<meerkat_comms::CommsRuntime>,
15262 supervisor_runtime: Arc<meerkat_comms::CommsRuntime>,
15263 supervisor_spec: TrustedPeerDescriptor,
15264 incarnation: BridgeMemberIncarnation,
15265 }
15266
15267 impl LiveArmFixture {
15268 async fn bound(label: &str) -> Self {
15272 let suffix = Uuid::new_v4().simple().to_string();
15273 let member_name = format!("live-{label}-member-{suffix}");
15274 let supervisor_name = format!("live-{label}-supervisor-{suffix}");
15275
15276 let mut member_runtime =
15277 meerkat_comms::CommsRuntime::inproc_only(&member_name).expect("member runtime");
15278 member_runtime
15279 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from((
15280 [127, 0, 0, 1],
15281 0,
15282 )))
15283 .expect("configure member TCP listener");
15284 member_runtime
15285 .start_listeners()
15286 .await
15287 .expect("start member TCP listener");
15288 let member_runtime = Arc::new(member_runtime);
15289 install_test_peer_request_response_authority(
15290 &member_runtime,
15291 Arc::new(CountingPeerInteractionHandle::default()),
15292 );
15293
15294 let mut supervisor_runtime = meerkat_comms::CommsRuntime::inproc_only(&supervisor_name)
15295 .expect("supervisor runtime");
15296 supervisor_runtime
15297 .set_listen_tcp_for_unstarted_runtime(std::net::SocketAddr::from((
15298 [127, 0, 0, 1],
15299 0,
15300 )))
15301 .expect("configure supervisor TCP listener");
15302 supervisor_runtime
15303 .start_listeners()
15304 .await
15305 .expect("start supervisor TCP listener");
15306 let supervisor_runtime = Arc::new(supervisor_runtime);
15307 install_test_peer_request_response_authority(
15308 &supervisor_runtime,
15309 Arc::new(CountingPeerInteractionHandle::default()),
15310 );
15311
15312 let member_spec = trusted_tcp_peer_from_runtime(&member_name, &member_runtime);
15313 let supervisor_dsl =
15314 install_running_test_peer_comms_handle(supervisor_runtime.as_ref());
15315 add_test_projection_trust_with_dsl(
15316 supervisor_runtime.as_ref(),
15317 Arc::clone(&supervisor_dsl),
15318 member_spec.clone(),
15319 1,
15320 "supervisor trusts live member target",
15321 )
15322 .await;
15323 let supervisor_spec =
15324 trusted_tcp_peer_from_runtime("mob/__mob_supervisor__", &supervisor_runtime);
15325
15326 let adapter = Arc::new(MeerkatMachine::ephemeral());
15327 let session_id = SessionId::new();
15328 adapter
15329 .register_session(session_id.clone())
15330 .await
15331 .expect("register session");
15332 adapter
15333 .stage_supervisor_bind(
15334 &session_id,
15335 supervisor_spec.name.to_string(),
15336 supervisor_spec.peer_id.as_str(),
15337 supervisor_spec.address.to_string(),
15338 signing_public_key_for_descriptor(&supervisor_spec),
15339 1,
15340 )
15341 .await
15342 .expect("pre-bind supervisor in DSL state");
15343 adapter
15344 .test_install_session_peer_comms_handle_on_runtime(
15345 &session_id,
15346 member_runtime.as_ref(),
15347 )
15348 .await
15349 .expect("install adapter session peer-comms handle on member runtime");
15350 add_member_private_supervisor_trust_via_adapter(
15354 &adapter,
15355 &session_id,
15356 &member_runtime,
15357 &supervisor_spec,
15358 1,
15359 "member starts with supervisor private route",
15360 )
15361 .await;
15362 let incarnation = BridgeMemberIncarnation {
15363 mob_id: "mob".to_string(),
15364 agent_identity: format!("live-{label}-member"),
15365 host_id: "host-a".to_string(),
15366 binding_generation: 1,
15367 member_session_id: session_id.to_string(),
15368 generation: 1,
15369 fence_token: 1,
15370 };
15371 register_member_incarnation(&adapter, session_id.clone(), incarnation.clone())
15372 .await
15373 .expect("register live fixture member incarnation");
15374
15375 Self {
15376 adapter,
15377 session_id,
15378 member_runtime,
15379 supervisor_runtime,
15380 supervisor_spec,
15381 incarnation,
15382 }
15383 }
15384
15385 fn candidate_for(&self, command: &BridgeCommand) -> PeerInputCandidate {
15386 let interaction_id = InteractionId(Uuid::new_v4());
15387 let ingress = PeerIngressFact::peer(
15388 interaction_id,
15389 PeerInputClass::ActionableRequest,
15390 meerkat_core::PeerIngressKind::Request,
15391 Some(meerkat_core::PeerIngressAuthDecision::Required),
15392 PeerIngressIdentity::new(
15393 self.supervisor_spec.peer_id,
15394 self.supervisor_spec.name.as_str(),
15395 meerkat_core::PeerIngressConvention::Request {
15396 request_id: interaction_id.to_string(),
15397 intent: SUPERVISOR_BRIDGE_INTENT.to_string(),
15398 },
15399 )
15400 .with_signing_pubkey(self.supervisor_spec.pubkey),
15401 );
15402 bridge_candidate_with_ingress(self.supervisor_spec.name.as_str(), command, ingress)
15403 }
15404
15405 fn supervisor_payload(&self) -> (BridgePeerSpec, u64, BridgeProtocolVersion) {
15406 (
15407 BridgePeerSpec::from(self.supervisor_spec.clone()),
15408 1,
15409 SUPERVISOR_BRIDGE_PROTOCOL_VERSION,
15410 )
15411 }
15412
15413 async fn serve(&self, command: &BridgeCommand) {
15414 let candidate = self.candidate_for(command);
15415 assert!(
15416 try_handle_supervisor_bridge_command(
15417 &self.adapter,
15418 &self.session_id,
15419 &(Arc::clone(&self.member_runtime) as Arc<dyn CommsRuntime>),
15420 &candidate,
15421 )
15422 .await,
15423 "bridge handler must own the live command"
15424 );
15425 }
15426
15427 async fn next_reply(&self, label: &str) -> BridgeReply {
15428 let response = drain_one_candidate(&self.supervisor_runtime, label).await;
15429 let InteractionContent::Response { result, .. } = response.interaction.content else {
15430 panic!("expected bridge response for {label}");
15431 };
15432 serde_json::from_value::<BridgeReply>(result).expect("typed bridge reply")
15433 }
15434 }
15435
15436 #[tokio::test]
15441 async fn member_live_verbs_round_trip_payload_to_trait_to_reply() {
15442 let fixture = LiveArmFixture::bound("roundtrip").await;
15443 let live = Arc::new(ScriptedMemberLive::default());
15444 fixture
15445 .adapter
15446 .set_member_live_host(Arc::clone(&live) as Arc<dyn MemberLiveHost>);
15447 let (supervisor, epoch, protocol_version) = fixture.supervisor_payload();
15448
15449 fixture
15450 .serve(&BridgeCommand::OpenMemberLiveChannel(
15451 BridgeLiveOpenPayload {
15452 supervisor: supervisor.clone(),
15453 epoch,
15454 protocol_version,
15455 expected_member: fixture.incarnation.clone(),
15456 turning_mode: None,
15457 transport: None,
15458 },
15459 ))
15460 .await;
15461 match fixture.next_reply("open reply").await {
15462 BridgeReply::MemberLiveChannelOpened(response) => {
15463 assert_eq!(
15464 response.open,
15465 scripted_open_result(),
15466 "LiveOpenResult must embed VERBATIM (gotcha #14): URL + token untouched"
15467 );
15468 }
15469 other => panic!("expected MemberLiveChannelOpened, got {other:?}"),
15470 }
15471
15472 fixture
15473 .serve(&BridgeCommand::MemberLiveChannelStatus(
15474 BridgeLiveStatusPayload {
15475 supervisor: supervisor.clone(),
15476 epoch,
15477 protocol_version,
15478 expected_member: fixture.incarnation.clone(),
15479 channel_id: None,
15480 },
15481 ))
15482 .await;
15483 match fixture.next_reply("status reply").await {
15484 BridgeReply::MemberLiveChannelStatusReport { channel_id, status } => {
15485 assert_eq!(channel_id, "chan-scripted");
15486 assert_eq!(status, WireLiveAdapterStatus::Ready);
15487 }
15488 other => panic!("expected MemberLiveChannelStatusReport, got {other:?}"),
15489 }
15490
15491 fixture
15492 .serve(&BridgeCommand::ControlMemberLiveChannel(
15493 BridgeLiveControlPayload {
15494 supervisor: supervisor.clone(),
15495 epoch,
15496 protocol_version,
15497 expected_member: fixture.incarnation.clone(),
15498 channel_id: "chan-scripted".to_string(),
15499 verb: BridgeLiveControlVerb::CommitInput,
15500 },
15501 ))
15502 .await;
15503 match fixture.next_reply("control reply").await {
15504 BridgeReply::MemberLiveChannelControlled(response) => {
15505 assert_eq!(
15506 response.outcome,
15507 BridgeLiveControlOutcome::CommitInput {
15508 status: LiveCommitInputStatus::Committed,
15509 }
15510 );
15511 }
15512 other => panic!("expected MemberLiveChannelControlled, got {other:?}"),
15513 }
15514
15515 fixture
15516 .serve(&BridgeCommand::CloseMemberLiveChannel(
15517 BridgeLiveChannelPayload {
15518 supervisor,
15519 epoch,
15520 protocol_version,
15521 expected_member: fixture.incarnation.clone(),
15522 channel_id: "chan-scripted".to_string(),
15523 },
15524 ))
15525 .await;
15526 match fixture.next_reply("close reply").await {
15527 BridgeReply::MemberLiveChannelClosed { status } => {
15528 assert_eq!(status, LiveCloseStatus::Closed);
15529 }
15530 other => panic!("expected MemberLiveChannelClosed, got {other:?}"),
15531 }
15532
15533 let calls = live.recorded();
15534 assert_eq!(
15535 calls,
15536 vec![
15537 ScriptedLiveCall::Open {
15538 session: fixture.session_id.clone(),
15539 turning_mode: None,
15540 transport: None,
15541 },
15542 ScriptedLiveCall::Status {
15543 session: fixture.session_id.clone(),
15544 channel_id: None,
15545 },
15546 ScriptedLiveCall::Control {
15547 session: fixture.session_id.clone(),
15548 channel_id: "chan-scripted".to_string(),
15549 verb: BridgeLiveControlVerb::CommitInput,
15550 },
15551 ScriptedLiveCall::Close {
15552 session: fixture.session_id.clone(),
15553 channel_id: "chan-scripted".to_string(),
15554 },
15555 ],
15556 "every arm must reach the trait with the drain's bound member session"
15557 );
15558 }
15559
15560 #[tokio::test]
15563 async fn unauthorized_supervisor_never_reaches_member_live_host() {
15564 let fixture = LiveArmFixture::bound("unauthorized").await;
15565 let live = Arc::new(ScriptedMemberLive::default());
15566 fixture
15567 .adapter
15568 .set_member_live_host(Arc::clone(&live) as Arc<dyn MemberLiveHost>);
15569 let (supervisor, _epoch, protocol_version) = fixture.supervisor_payload();
15570
15571 fixture
15573 .serve(&BridgeCommand::CloseMemberLiveChannel(
15574 BridgeLiveChannelPayload {
15575 supervisor,
15576 epoch: 0,
15577 protocol_version,
15578 expected_member: fixture.incarnation.clone(),
15579 channel_id: "chan-any".to_string(),
15580 },
15581 ))
15582 .await;
15583 match fixture.next_reply("unauthorized close reply").await {
15584 BridgeReply::Rejected { .. } => {}
15585 other => panic!("expected typed rejection, got {other:?}"),
15586 }
15587 assert!(
15588 live.recorded().is_empty(),
15589 "an unauthorized command must never reach the member live host"
15590 );
15591 }
15592
15593 #[tokio::test]
15597 async fn absent_member_live_slot_rejects_live_transport_unavailable() {
15598 let fixture = LiveArmFixture::bound("absent-slot").await;
15599 let (supervisor, epoch, protocol_version) = fixture.supervisor_payload();
15600
15601 fixture
15602 .serve(&BridgeCommand::CloseMemberLiveChannel(
15603 BridgeLiveChannelPayload {
15604 supervisor,
15605 epoch,
15606 protocol_version,
15607 expected_member: fixture.incarnation.clone(),
15608 channel_id: "chan-any".to_string(),
15609 },
15610 ))
15611 .await;
15612 match fixture.next_reply("absent-slot close reply").await {
15613 BridgeReply::Rejected { cause, reason } => {
15614 assert_eq!(cause, BridgeRejectionCause::LiveTransportUnavailable);
15615 assert_eq!(reason, "member host serves no live substrate");
15616 }
15617 other => panic!("expected typed rejection, got {other:?}"),
15618 }
15619 }
15620
15621 #[test]
15624 fn resolve_member_live_host_absent_is_live_transport_unavailable() {
15625 let adapter = Arc::new(MeerkatMachine::ephemeral());
15626 let Err((cause, reason)) = resolve_member_live_host(&adapter) else {
15627 panic!("empty slot must reject");
15628 };
15629 assert_eq!(cause, BridgeRejectionCause::LiveTransportUnavailable);
15630 assert_eq!(reason, "member host serves no live substrate");
15631 }
15632
15633 #[test]
15638 fn member_live_error_rejection_uses_shared_conversion_and_display_reason() {
15639 let error = MemberLiveError::TransportUnsupported {
15640 requested: "webrtc".to_string(),
15641 };
15642 let (cause, reason) = member_live_error_to_bridge_rejection(&error);
15643 assert_eq!(cause, error.to_bridge_rejection());
15644 assert_eq!(reason, error.to_string());
15645
15646 let internal = MemberLiveError::Internal {
15647 reason: "invariant".to_string(),
15648 };
15649 let (cause, reason) = member_live_error_to_bridge_rejection(&internal);
15650 assert_eq!(cause, BridgeRejectionCause::Internal);
15651 assert_eq!(reason, "member live internal fault: invariant");
15652 }
15653
15654 #[tokio::test]
15658 async fn slow_member_live_open_does_not_block_inline_status() {
15659 let fixture = LiveArmFixture::bound("detached-open").await;
15660 let gate = Arc::new(tokio::sync::Semaphore::new(0));
15661 let live = Arc::new(ScriptedMemberLive {
15662 calls: std::sync::Mutex::new(Vec::new()),
15663 open_gate: Some(Arc::clone(&gate)),
15664 });
15665 fixture
15666 .adapter
15667 .set_member_live_host(Arc::clone(&live) as Arc<dyn MemberLiveHost>);
15668 let (supervisor, epoch, protocol_version) = fixture.supervisor_payload();
15669
15670 fixture
15673 .serve(&BridgeCommand::OpenMemberLiveChannel(
15674 BridgeLiveOpenPayload {
15675 supervisor: supervisor.clone(),
15676 epoch,
15677 protocol_version,
15678 expected_member: fixture.incarnation.clone(),
15679 turning_mode: None,
15680 transport: None,
15681 },
15682 ))
15683 .await;
15684
15685 fixture
15688 .serve(&BridgeCommand::MemberLiveChannelStatus(
15689 BridgeLiveStatusPayload {
15690 supervisor,
15691 epoch,
15692 protocol_version,
15693 expected_member: fixture.incarnation.clone(),
15694 channel_id: None,
15695 },
15696 ))
15697 .await;
15698 match fixture.next_reply("status while open parked").await {
15699 BridgeReply::MemberLiveChannelStatusReport { .. } => {}
15700 other => panic!("expected status report before the gated open reply, got {other:?}"),
15701 }
15702
15703 gate.add_permits(1);
15705 match fixture.next_reply("gated open reply").await {
15706 BridgeReply::MemberLiveChannelOpened(response) => {
15707 assert_eq!(response.open, scripted_open_result());
15708 }
15709 other => panic!("expected MemberLiveChannelOpened, got {other:?}"),
15710 }
15711 }
15712}
15713
15714fn spawn_completion_bridge(
15716 comms_runtime: Option<Arc<dyn CommsRuntime>>,
15717 interaction_id: meerkat_core::interaction::InteractionId,
15718 subscriber: Option<mpsc::Sender<AgentEvent>>,
15719 handle: Option<crate::completion::CompletionHandle>,
15720) {
15721 crate::tokio::spawn(async move {
15722 let delivered_terminal = if let Some(handle) = handle {
15723 match handle.try_wait().await {
15724 Ok(outcome) => {
15725 if let Some(tx) = subscriber {
15726 let event = interaction_terminal_event(interaction_id, outcome);
15727 match crate::tokio::time::timeout(
15728 std::time::Duration::from_secs(5),
15729 tx.send(event),
15730 )
15731 .await
15732 {
15733 Ok(Ok(())) => true,
15734 Ok(Err(_)) => {
15735 tracing::warn!(
15736 %interaction_id,
15737 "completion bridge could not deliver terminal event: subscriber closed"
15738 );
15739 false
15740 }
15741 Err(_) => {
15742 tracing::warn!(
15743 %interaction_id,
15744 "completion bridge could not deliver terminal event: subscriber send timed out after 5s"
15745 );
15746 false
15747 }
15748 }
15749 } else {
15750 tracing::warn!(
15751 %interaction_id,
15752 "completion bridge has no interaction subscriber for terminal event"
15753 );
15754 false
15755 }
15756 }
15757 Err(error) => {
15758 tracing::warn!(
15759 %interaction_id,
15760 error = %error,
15761 "completion bridge waiter failed without generated completion authority; not fabricating terminal event"
15762 );
15763 false
15764 }
15765 }
15766 } else {
15767 tracing::debug!(
15768 %interaction_id,
15769 "completion bridge has no completion handle; not fabricating terminal event"
15770 );
15771 false
15772 };
15773
15774 if !delivered_terminal && let Some(runtime) = comms_runtime {
15775 runtime.abandon_interaction_stream(
15776 &interaction_id,
15777 meerkat_core::InteractionStreamAbandonReason::TerminalDeliveryFailed,
15778 );
15779 }
15780 });
15781}