actr-hyper 0.3.0

Hyper — Actor platform infrastructure: sandbox, transport, scheduler, WASM engine, signing, AIS bootstrap, persistence & crypto primitives
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
//! Test Harness for PeerGate integration tests
//!
//! Provides `TestPeer` and `TestHarness` for building multi-peer test
//! topologies with optional VNet-based network simulation.
//!
//! ## Usage
//!
//! ```rust,ignore
//! // Basic two-peer test
//! let mut harness = TestHarness::new().await;
//! harness.add_peer(100).await;
//! harness.add_peer(200).await;
//! harness.connect(100, 200).await;
//!
//! // Multi-peer with VNet (disconnect simulation)
//! let mut harness = TestHarness::with_vnet().await;
//! harness.add_peer(100).await;
//! harness.add_peer(200).await;
//! harness.add_peer(300).await;
//! harness.connect(200, 100).await;
//! harness.connect(300, 100).await;
//!
//! harness.simulate_disconnect();
//! // ... verify ICE restart triggered
//! harness.simulate_reconnect();
//! ```

use super::signaling::TestSignalingServer;
use super::utils::{
    create_peer_with_vnet, create_peer_with_websocket, make_actor_id, spawn_echo_responder,
    spawn_response_receiver,
};
use super::vnet::VNetPair;
use crate::lifecycle::DefaultNetworkEventProcessor;
use crate::outbound::PeerGate;
use crate::transport::{DefaultWireBuilder, DefaultWireBuilderConfig, PeerTransport};
use crate::wire::webrtc::{SignalingClient, WebRtcCoordinator};
use actr_protocol::{ActrId, RpcEnvelope};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;

/// A single test peer encapsulating all components needed for peer communication.
pub struct TestPeer {
    /// Actor ID for this peer
    pub id: ActrId,
    /// WebRTC coordinator (connection management, ICE restart, signaling)
    pub coordinator: Arc<WebRtcCoordinator>,
    /// Signaling client used by network event processors in mobile-style tests.
    pub signaling_client: Arc<dyn SignalingClient>,
    /// PeerGate (message sending, pending request management)
    pub gate: Arc<PeerGate>,
    /// Transport manager (wire pool, dest transport management)
    pub transport_manager: Arc<PeerTransport>,
}

impl TestPeer {
    /// Subscribe to connection events from this peer's coordinator
    pub fn subscribe_events(
        &self,
    ) -> tokio::sync::broadcast::Receiver<crate::transport::ConnectionEvent> {
        self.coordinator.subscribe_events()
    }

    /// Get pending request count
    pub async fn pending_count(&self) -> usize {
        self.gate.pending_count().await
    }

    /// Trigger ICE restart to a target peer
    pub async fn restart_ice(&self, target_serial: u64) -> anyhow::Result<()> {
        let target_id = make_actor_id(target_serial);
        self.coordinator
            .restart_ice(&target_id)
            .await
            .map_err(|e| anyhow::anyhow!("ICE restart failed: {}", e))
    }

    /// Retry failed connections
    pub async fn retry_failed(&self) {
        self.coordinator.retry_failed_connections().await;
    }

    /// Create a network event processor for this peer.
    pub fn network_processor(&self) -> Arc<DefaultNetworkEventProcessor> {
        Arc::new(DefaultNetworkEventProcessor::new(
            self.signaling_client.clone(),
            Some(self.coordinator.clone()),
        ))
    }

    /// Send a test RPC request to a target peer (fire-and-forget, returns handle)
    pub fn spawn_request(
        &self,
        target_serial: u64,
        request_id: &str,
        timeout_ms: u32,
    ) -> tokio::task::JoinHandle<actr_protocol::ActorResult<actr_framework::Bytes>> {
        let gate = self.gate.clone();
        let target_id = make_actor_id(target_serial);
        let envelope = RpcEnvelope {
            request_id: request_id.to_string(),
            route_key: "test.method".to_string(),
            payload: Some(bytes::Bytes::from("test_payload")),
            timeout_ms: timeout_ms as i64,
            ..Default::default()
        };
        tokio::spawn(async move { gate.send_request(&target_id, envelope).await })
    }

    /// Send a ConnectionEvent to simulate state changes
    pub fn send_event(&self, event: crate::transport::ConnectionEvent) {
        let _ = self.coordinator.event_sender().send(event);
    }

    /// Start an echo responder on this peer.
    ///
    /// Receives RPC requests from the coordinator and sends back "pong" responses.
    /// Call this on the **target** peer before sending requests.
    pub fn start_echo_responder(&self, name: &str) -> tokio::task::JoinHandle<()> {
        spawn_echo_responder(self.coordinator.clone(), self.gate.clone(), name)
    }

    /// Start a response receiver on this peer.
    ///
    /// Receives RPC responses from the coordinator and routes them to
    /// `gate.handle_response()` to wake up pending requests.
    /// Call this on the **source** peer before sending requests.
    pub fn start_response_receiver(&self, name: &str) -> tokio::task::JoinHandle<()> {
        spawn_response_receiver(self.coordinator.clone(), self.gate.clone(), name)
    }
}

/// Test harness supporting dynamic multi-peer topologies with optional VNet.
///
/// Manages a signaling server, optional VNet pair, and a collection of `TestPeer`s
/// indexed by their serial number.
pub struct TestHarness {
    /// Shared signaling server
    pub server: TestSignalingServer,
    /// Optional VNet pair for network simulation
    pub vnet: Option<VNetPair>,
    /// Peers indexed by serial_number
    peers: HashMap<u64, TestPeer>,
    /// Background task handles (echo responders, response receivers)
    _bg_tasks: Vec<tokio::task::JoinHandle<()>>,
}

impl TestHarness {
    /// Create a new TestHarness without VNet (no network simulation).
    pub async fn new() -> Self {
        let server = TestSignalingServer::start()
            .await
            .expect("Failed to start signaling server");
        Self {
            server,
            vnet: None,
            peers: HashMap::new(),
            _bg_tasks: Vec::new(),
        }
    }

    /// Create a new TestHarness with VNet for network disconnection simulation.
    pub async fn with_vnet() -> Self {
        let server = TestSignalingServer::start()
            .await
            .expect("Failed to start signaling server");
        let vnet = VNetPair::new().await.expect("Failed to create VNet pair");
        Self {
            server,
            vnet: Some(vnet),
            peers: HashMap::new(),
            _bg_tasks: Vec::new(),
        }
    }

    /// Add a peer with the given serial number.
    ///
    /// If VNet is enabled, the peer is assigned to offerer or answerer network
    /// based on addition order (first = offerer, rest = answerer).
    /// This supports the common topology of "multiple peers connecting to one hub".
    ///
    /// # Arguments
    /// - `serial`: Serial number for the ActrId (used as peer key)
    pub async fn add_peer(&mut self, serial: u64) {
        assert!(
            !self.peers.contains_key(&serial),
            "Peer with serial {} already exists",
            serial
        );

        let id = make_actor_id(serial);
        let server_url = self.server.url();

        let (coordinator, signaling_client) = if let Some(ref vnet) = self.vnet {
            // With VNet: assign first peer to offerer net, rest to answerer net
            let net = if self.peers.is_empty() {
                vnet.net_offerer.clone()
            } else {
                vnet.net_answerer.clone()
            };
            create_peer_with_vnet(id.clone(), &server_url, net)
                .await
                .expect("Failed to create peer with vnet")
        } else {
            // Without VNet: standard WebSocket connection
            create_peer_with_websocket(id.clone(), &server_url)
                .await
                .expect("Failed to create peer")
        };

        // Build PeerGate with full transport stack
        let wire_config = DefaultWireBuilderConfig::default();
        let wire_builder = Arc::new(DefaultWireBuilder::new(
            Some(coordinator.clone()),
            wire_config,
        ));
        let transport_manager = Arc::new(PeerTransport::new(id.clone(), wire_builder));
        let gate = Arc::new(PeerGate::new(
            transport_manager.clone(),
            Some(coordinator.clone()),
        ));

        self.peers.insert(
            serial,
            TestPeer {
                id,
                coordinator,
                signaling_client,
                gate,
                transport_manager,
            },
        );

        tracing::info!("✅ Added test peer with serial {}", serial);
    }

    /// Get a reference to a peer by serial number.
    ///
    /// # Panics
    /// Panics if the peer doesn't exist.
    pub fn peer(&self, serial: u64) -> &TestPeer {
        self.peers
            .get(&serial)
            .unwrap_or_else(|| panic!("Peer with serial {} not found", serial))
    }

    /// Get a mutable reference to a peer by serial number.
    pub fn peer_mut(&mut self, serial: u64) -> &mut TestPeer {
        self.peers
            .get_mut(&serial)
            .unwrap_or_else(|| panic!("Peer with serial {} not found", serial))
    }

    /// Get all peer serial numbers.
    pub fn peer_serials(&self) -> Vec<u64> {
        self.peers.keys().copied().collect()
    }

    /// Get total peer count.
    pub fn peer_count(&self) -> usize {
        self.peers.len()
    }

    /// Establish a connection from one peer to another **by sending a message
    /// through the PeerGate**.
    ///
    /// This triggers the full transport stack:
    /// `PeerGate → PeerTransport (lazy create) → WireBuilder → WebRTC`
    ///
    /// Internally:
    /// 1. Starts an **echo responder** on the target peer
    /// 2. Starts a **response receiver** on the source peer
    /// 3. Sends a test RPC request through the source peer's gate
    /// 4. The transport manager lazily creates the WebRTC connection
    /// 5. The echo responder sends back a "pong" response
    /// 6. Request success = end-to-end connection verified
    ///
    /// # Arguments
    /// - `from_serial`: Initiating peer's serial number
    /// - `to_serial`: Target peer's serial number
    pub async fn connect(&mut self, from_serial: u64, to_serial: u64) {
        self.connect_with_timeout(from_serial, to_serial, Duration::from_secs(15))
            .await;
    }

    /// Establish a connection with custom timeout (via gate message).
    pub async fn connect_with_timeout(
        &mut self,
        from_serial: u64,
        to_serial: u64,
        timeout: Duration,
    ) {
        // Extract Arc handles first to avoid borrow overlap with self._bg_tasks
        let (from_coord, from_gate, to_coord, to_gate, target_id) = {
            let from_peer = self.peer(from_serial);
            let to_peer = self.peer(to_serial);
            (
                from_peer.coordinator.clone(),
                from_peer.gate.clone(),
                to_peer.coordinator.clone(),
                to_peer.gate.clone(),
                to_peer.id.clone(),
            )
        };

        tracing::info!(
            "🔗 Connecting peer {} → peer {} (via gate message)...",
            from_serial,
            to_serial
        );

        // 1. Start echo responder on target peer (receives requests, sends "pong")
        let echo_handle = spawn_echo_responder(to_coord, to_gate, &format!("echo_{}", to_serial));
        self._bg_tasks.push(echo_handle);

        // 2. Start response receiver on source peer (routes responses to gate.handle_response)
        let recv_handle = spawn_response_receiver(
            from_coord,
            from_gate.clone(),
            &format!("recv_{}", from_serial),
        );
        self._bg_tasks.push(recv_handle);

        // 3. Send a test request through the gate — this triggers lazy connection creation
        let request_id = format!("connect_test_{}_{}", from_serial, to_serial);

        let envelope = RpcEnvelope {
            request_id: request_id.clone(),
            route_key: "test.ping".to_string(),
            payload: Some(bytes::Bytes::from("ping")),
            timeout_ms: timeout.as_millis() as i64,
            ..Default::default()
        };

        match tokio::time::timeout(timeout, from_gate.send_request(&target_id, envelope)).await {
            Ok(Ok(response)) => {
                tracing::info!(
                    "✅ Connection established and verified: {} → {} (response: {} bytes)",
                    from_serial,
                    to_serial,
                    response.len()
                );
            }
            Ok(Err(e)) => panic!("Connection {} → {} failed: {}", from_serial, to_serial, e),
            Err(_) => panic!(
                "Connection {} → {} timed out after {:?}",
                from_serial, to_serial, timeout
            ),
        }

        // Brief stabilization delay
        tokio::time::sleep(Duration::from_millis(300)).await;
    }

    /// Block all network traffic AND pause signaling (simulate full network outage).
    ///
    /// This simulates a real-world disconnection where:
    /// - UDP traffic is blocked (VNet) → ICE connectivity checks fail
    /// - Signaling messages stop forwarding → ICE restart offers can't reach the peer
    ///
    /// # Panics
    /// Panics if VNet is not enabled.
    pub fn simulate_disconnect(&self) {
        let vnet = self
            .vnet
            .as_ref()
            .expect("simulate_disconnect requires VNet (use TestHarness::with_vnet())");
        tracing::warn!("🔴 Simulating full network disconnection (VNet + signaling)");
        vnet.block_network();
        self.server.pause_forwarding();
    }

    /// Unblock network traffic AND resume signaling (simulate network recovery).
    ///
    /// # Panics
    /// Panics if VNet is not enabled.
    pub fn simulate_reconnect(&self) {
        let vnet = self
            .vnet
            .as_ref()
            .expect("simulate_reconnect requires VNet (use TestHarness::with_vnet())");
        tracing::info!("🟢 Simulating network recovery (VNet + signaling)");
        self.server.resume_forwarding();
        vnet.unblock_network();
    }

    /// Check if network is currently blocked.
    pub fn is_disconnected(&self) -> bool {
        self.vnet.as_ref().is_some_and(|v| v.is_blocked())
    }

    /// Wait until the signaling server's ICE restart count increases beyond `min_count`.
    ///
    /// # Arguments
    /// - `min_count`: Minimum expected ICE restart count
    /// - `timeout`: Maximum time to wait
    ///
    /// # Returns
    /// The final ICE restart count
    pub async fn wait_for_ice_restart_count(&self, min_count: u32, timeout: Duration) -> u32 {
        let deadline = tokio::time::Instant::now() + timeout;
        loop {
            let count = self.server.get_ice_restart_count();
            if count >= min_count {
                return count;
            }
            if tokio::time::Instant::now() >= deadline {
                panic!(
                    "Timed out waiting for ICE restart count >= {} (current: {})",
                    min_count, count
                );
            }
            tokio::time::sleep(Duration::from_millis(100)).await;
        }
    }

    /// Wait until the signaling server's ICE restart request count reaches `min_count`.
    pub async fn wait_for_ice_restart_request_count(
        &self,
        min_count: u32,
        timeout: Duration,
    ) -> u32 {
        let deadline = tokio::time::Instant::now() + timeout;
        loop {
            let count = self.server.get_ice_restart_request_count();
            if count >= min_count {
                return count;
            }
            if tokio::time::Instant::now() >= deadline {
                panic!(
                    "Timed out waiting for ICE restart request count >= {} (current: {})",
                    min_count, count
                );
            }
            tokio::time::sleep(Duration::from_millis(100)).await;
        }
    }

    /// Reset the signaling server's counters.
    pub fn reset_counters(&self) {
        self.server.reset_counters();
    }

    /// Get current ICE restart count from the signaling server.
    pub fn ice_restart_count(&self) -> u32 {
        self.server.get_ice_restart_count()
    }
}