Skip to main content

infinity_daemon/
remote.rs

1use std::collections::HashMap;
2use std::sync::Arc;
3
4use bytes::Bytes;
5use futures_util::{SinkExt, StreamExt};
6use infinity_protocol::{ClientMessage, DaemonMessage, SessionInfo, length_delimited_codec};
7use serde::Deserialize;
8use tokio::net::UnixStream;
9use tokio::process::Child;
10use tokio::sync::mpsc;
11use tokio_util::codec::{Framed, LengthDelimitedCodec};
12
13type BoxError = Box<dyn std::error::Error + Send + Sync>;
14pub type BroadcastClients = Arc<std::sync::Mutex<Vec<mpsc::UnboundedSender<DaemonMessage>>>>;
15
16#[derive(serde::Serialize, Deserialize, Clone, Debug)]
17pub struct RemoteConfig {
18    pub name: infinity_protocol::RemoteName,
19    pub ssh_args: Vec<String>,
20}
21
22pub fn load_remotes_config() -> Vec<RemoteConfig> {
23    let path = infinity_protocol::remotes_config_path();
24    match std::fs::read_to_string(&path) {
25        Ok(contents) => serde_json::from_str(&contents).unwrap_or_default(),
26        Err(_) => Vec::new(),
27    }
28}
29
30#[derive(Debug, Clone)]
31pub enum RemoteStatus {
32    Connecting,
33    Connected,
34    Disconnected(String),
35}
36
37pub struct RemoteState {
38    pub status: RemoteStatus,
39    pub sessions: HashMap<infinity_protocol::ThreadRef, SessionInfo>,
40    /// Local socket path for the SSH tunnel (set when connected).
41    pub local_sock: Option<std::path::PathBuf>,
42    /// SSH args for this remote.
43    pub ssh_args: Vec<String>,
44}
45
46type RemoteMap = Arc<std::sync::Mutex<HashMap<infinity_protocol::RemoteName, RemoteState>>>;
47
48#[derive(Clone)]
49pub struct RemoteDaemons {
50    remotes: RemoteMap,
51}
52
53impl RemoteDaemons {
54    pub fn new(configs: Vec<RemoteConfig>, broadcast_clients: BroadcastClients) -> Self {
55        let remotes: RemoteMap = Arc::new(std::sync::Mutex::new(HashMap::new()));
56        for cfg in &configs {
57            remotes.lock().expect("bug: mutex poisoned").insert(
58                cfg.name.clone(),
59                RemoteState {
60                    status: RemoteStatus::Connecting,
61                    sessions: HashMap::new(),
62                    local_sock: None,
63                    ssh_args: cfg.ssh_args.clone(),
64                },
65            );
66            let name = cfg.name.clone();
67            let ssh_args = cfg.ssh_args.clone();
68            let map = remotes.clone();
69            let bc = broadcast_clients.clone();
70            tokio::task::spawn_local(rap_protocol::log_panic(
71                "remote_control_worker",
72                control_worker(name, ssh_args, map, bc),
73            ));
74        }
75        Self { remotes }
76    }
77
78    /// Collect all remote sessions with prefixed IDs (used for initial Welcome).
79    pub fn all_remote_sessions(&self) -> HashMap<infinity_protocol::ThreadRef, SessionInfo> {
80        let map = self.remotes.lock().expect("bug: mutex poisoned");
81        let mut result = HashMap::new();
82        for (remote_name, state) in map.iter() {
83            if !matches!(state.status, RemoteStatus::Connected) {
84                continue;
85            }
86            for (sid, info) in &state.sessions {
87                let mut info = info.clone();
88                info.remote = Some(remote_name.clone());
89                for t in &mut info.threads {
90                    t.thread_id = t.thread_id.clone().prefixed(remote_name);
91                    t.parent_thread_id = t.parent_thread_id.clone().prefixed(remote_name);
92                }
93                result.insert(sid.clone().prefixed(remote_name), info);
94            }
95        }
96        result
97    }
98
99    /// Open a new proxied connection to a remote session.
100    /// Reuses the existing SSH tunnel socket when available.
101    pub async fn connect_remote_session(
102        &self,
103        remote_name: &infinity_protocol::RemoteName,
104        session_id: &infinity_agent_core::ThreadId<str>,
105        thread_id: Option<&infinity_agent_core::ThreadId<str>>,
106    ) -> Result<
107        (
108            mpsc::UnboundedSender<ClientMessage>,
109            mpsc::UnboundedReceiver<DaemonMessage>,
110        ),
111        BoxError,
112    > {
113        let res = self.open_raw_connection(remote_name).await?;
114
115        res.0
116            .send(ClientMessage::Connect {
117                root_thread_id: infinity_protocol::ThreadRef::local(session_id.to_owned()),
118                thread_id: thread_id.map(|t| infinity_protocol::ThreadRef::local(t.to_owned())),
119                keeps_session_alive: true,
120            })
121            .map_err(|e| format!("send Connect failed: {e}"))?;
122
123        Ok(res)
124    }
125
126    /// Open a raw connection to a remote daemon (Welcome consumed, no Connect sent).
127    /// Used for migration flows where we send custom messages.
128    pub async fn open_raw_connection(
129        &self,
130        remote_name: &infinity_protocol::RemoteName,
131    ) -> Result<
132        (
133            mpsc::UnboundedSender<ClientMessage>,
134            mpsc::UnboundedReceiver<DaemonMessage>,
135        ),
136        BoxError,
137    > {
138        tracing::info!("Opening remote connection to {}", remote_name);
139
140        let local_sock = {
141            let map = self.remotes.lock().expect("bug: mutex poisoned");
142            map.get(remote_name)
143                .and_then(|s| s.local_sock.clone())
144                .ok_or_else(|| format!("remote '{remote_name}' is not connected"))?
145        };
146
147        let stream = UnixStream::connect(&local_sock)
148            .await
149            .map_err(|e| format!("connect to tunnel failed: {e}"))?;
150        let mut framed = Framed::new(stream, length_delimited_codec());
151
152        // Read and discard Welcome
153        let first = framed
154            .next()
155            .await
156            .ok_or("remote closed before Welcome")?
157            .map_err(|e| format!("read error: {e}"))?;
158        let _welcome: DaemonMessage =
159            serde_json::from_slice(&first).map_err(|e| format!("invalid Welcome: {e}"))?;
160
161        let (client_tx, mut client_rx) = mpsc::unbounded_channel::<ClientMessage>();
162        let (daemon_tx, daemon_rx) = mpsc::unbounded_channel::<DaemonMessage>();
163
164        tokio::task::spawn_local(async move {
165            let (mut sink, mut stream) = framed.split();
166            loop {
167                tokio::select! {
168                    msg = stream.next() => {
169                        let bytes = match msg {
170                            Some(Ok(bytes)) => bytes,
171                            Some(Err(e)) => {
172                                tracing::error!("Error reading from remote connection: {e}");
173                                break;
174                            }
175                            None => {
176                                tracing::error!("Remote connection closed unexpectedly");
177                                break;
178                            }
179                        };
180                        let Ok(dm) = serde_json::from_slice::<DaemonMessage>(&bytes) else {
181                            tracing::error!("Failed to parse DaemonMessage from remote: {}", String::from_utf8_lossy(&bytes));
182                            break;
183                        };
184                        if matches!(dm, DaemonMessage::Welcome { .. } | DaemonMessage::SessionsUpdated { .. } | DaemonMessage::RemotesUpdated { .. }) {
185                            continue;
186                        }
187                        if daemon_tx.send(dm).is_err() {
188                            tracing::info!("Remote receiver disconnected, so closing connection");
189                            break;
190                        }
191                    }
192                    msg = client_rx.recv() => {
193                        let Some(msg) = msg else {
194                            tracing::info!("Client sender disconnected, so closing connection");
195                            break
196                        };
197                        let bytes = Bytes::from(serde_json::to_vec(&msg).expect("bug: serialization failed"));
198                        if sink.send(bytes).await.is_err() {
199                            tracing::error!("Error sending to remote connection");
200                            break;
201                        }
202                    }
203                }
204            }
205        });
206
207        Ok((client_tx, daemon_rx))
208    }
209
210    /// Get the SSH args for a remote by name.
211    pub fn get_ssh_args(&self, remote_name: &infinity_protocol::RemoteName) -> Option<Vec<String>> {
212        let map = self.remotes.lock().expect("bug: mutex poisoned");
213        map.get(remote_name).map(|s| s.ssh_args.clone())
214    }
215
216    /// Return the current status of all configured remotes.
217    pub fn remote_info_list(&self) -> Vec<infinity_protocol::RemoteInfo> {
218        build_remote_info_list(&self.remotes)
219    }
220}
221
222/// SSH-forward a remote port to a local port.
223/// Returns the local port and a guard that kills the SSH process on drop.
224pub async fn ssh_forward_port(
225    ssh_args: &[String],
226    remote_port: u16,
227) -> Result<(u16, SshPortForward), BoxError> {
228    use std::process::Stdio;
229
230    // Use -L 0:127.0.0.1:<remote_port> to get a dynamic local port
231    let mut child = tokio::process::Command::new("ssh")
232        .args(ssh_args)
233        .arg("-L")
234        .arg(format!("0:127.0.0.1:{remote_port}"))
235        .arg("-N")
236        .arg("-o")
237        .arg("ExitOnForwardFailure=yes")
238        .stdin(Stdio::null())
239        .stdout(Stdio::null())
240        .stderr(Stdio::piped())
241        .spawn()
242        .map_err(|e| format!("ssh forward spawn failed: {e}"))?;
243
244    // SSH with -L 0:... doesn't easily report the allocated port.
245    // Instead, use a known local port by binding a listener first to find a free one.
246    drop(child.stderr.take());
247
248    // Simpler approach: use a temp unix socket like the control worker does.
249    // Actually simplest: pick a free port ourselves.
250    let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
251        .await
252        .map_err(|e| format!("bind failed: {e}"))?;
253    let local_port = listener.local_addr()?.port();
254    drop(listener);
255
256    // Kill the dynamic one and re-spawn with the known port
257    let _ = child.kill().await;
258
259    let child = tokio::process::Command::new("ssh")
260        .args(ssh_args)
261        .arg("-L")
262        .arg(format!("{local_port}:127.0.0.1:{remote_port}"))
263        .arg("-N")
264        .arg("-o")
265        .arg("ExitOnForwardFailure=yes")
266        .stdin(Stdio::null())
267        .stdout(Stdio::null())
268        .stderr(Stdio::null())
269        .kill_on_drop(true)
270        .spawn()
271        .map_err(|e| format!("ssh forward spawn failed: {e}"))?;
272
273    // Wait briefly for the tunnel to establish
274    tokio::time::sleep(std::time::Duration::from_millis(500)).await;
275
276    Ok((local_port, SshPortForward { _child: child }))
277}
278
279/// SSH-reverse-forward a local port so it's reachable on the remote host.
280/// Returns the allocated port on the remote and a guard.
281pub async fn ssh_reverse_forward_port(
282    ssh_args: &[String],
283    local_port: u16,
284) -> Result<(u16, SshPortForward), BoxError> {
285    use std::process::Stdio;
286
287    tracing::info!("Reverse forwarding port from remote to localhost:{local_port}");
288
289    // -R 0:127.0.0.1:<local_port> asks the remote to allocate a port.
290    // We capture stderr to read the allocated port from OpenSSH's output.
291    // However, OpenSSH only prints this with -v and it's unreliable.
292    // Instead: pick a port on the remote by probing, or just use the same port number
293    // and hope it's free. Safest: use a known free port by asking the remote.
294    let remote_port = resolve_free_remote_port(ssh_args).await?;
295
296    let child = tokio::process::Command::new("ssh")
297        .args(ssh_args)
298        .arg("-R")
299        .arg(format!("{remote_port}:127.0.0.1:{local_port}"))
300        .arg("-N")
301        .arg("-o")
302        .arg("ExitOnForwardFailure=yes")
303        .stdin(Stdio::null())
304        .stdout(Stdio::null())
305        .stderr(Stdio::null())
306        .kill_on_drop(true)
307        .spawn()
308        .map_err(|e| format!("ssh reverse forward spawn failed: {e}"))?;
309
310    tokio::time::sleep(std::time::Duration::from_millis(500)).await;
311    tracing::info!(
312        "Successfully allocated remote port from remote:{remote_port} to localhost:{local_port}"
313    );
314
315    Ok((remote_port, SshPortForward { _child: child }))
316}
317
318/// Find a free port on the remote host.
319async fn resolve_free_remote_port(ssh_args: &[String]) -> Result<u16, BoxError> {
320    let output = tokio::process::Command::new("ssh")
321        .args(ssh_args)
322        .arg("--")
323        .arg("python3 -c 'import socket; s=socket.socket(); s.bind((\"\",0)); print(s.getsockname()[1]); s.close()'")
324        .output()
325        .await
326        .map_err(|e| format!("ssh failed: {e}"))?;
327    if !output.status.success() {
328        return Err("failed to find free remote port".into());
329    }
330    let port: u16 = String::from_utf8_lossy(&output.stdout)
331        .trim()
332        .parse()
333        .map_err(|e| format!("invalid port: {e}"))?;
334    Ok(port)
335}
336
337/// Guard that keeps an SSH port-forward process alive.
338pub struct SshPortForward {
339    _child: Child,
340}
341
342/// Prefix session IDs in a sessions map with the remote name.
343fn prefix_sessions(
344    name: &infinity_protocol::RemoteName,
345    sessions: HashMap<infinity_protocol::ThreadRef, SessionInfo>,
346) -> HashMap<infinity_protocol::ThreadRef, SessionInfo> {
347    sessions
348        .into_iter()
349        .map(|(id, mut info)| {
350            info.remote = Some(name.clone());
351            for t in &mut info.threads {
352                t.thread_id = t.thread_id.clone().prefixed(name);
353                t.parent_thread_id = t.parent_thread_id.clone().prefixed(name);
354            }
355            (id.prefixed(name), info)
356        })
357        .collect()
358}
359
360/// An open SSH-forwarded connection: holds the SSH child process and temp
361/// socket path so the tunnel stays alive and cleans up on drop.
362struct SshTunnel {
363    _child: Child,
364    local_sock: std::path::PathBuf,
365}
366
367impl Drop for SshTunnel {
368    fn drop(&mut self) {
369        let _ = std::fs::remove_file(&self.local_sock);
370    }
371}
372
373/// Resolve the remote daemon socket path by running `echo $HOME` over SSH.
374async fn resolve_remote_sock(ssh_args: &[String]) -> Result<String, BoxError> {
375    let output = tokio::process::Command::new("ssh")
376        .args(ssh_args)
377        .arg("--")
378        .arg("echo $HOME/.infinity/daemon.sock")
379        .output()
380        .await
381        .map_err(|e| format!("ssh spawn failed: {e}"))?;
382    if !output.status.success() {
383        return Err(format!(
384            "ssh echo failed: {}",
385            String::from_utf8_lossy(&output.stderr)
386        )
387        .into());
388    }
389    Ok(String::from_utf8_lossy(&output.stdout).trim().to_owned())
390}
391
392/// Open an SSH `-L` tunnel to the remote daemon's unix socket and connect.
393async fn open_remote_connection(
394    ssh_args: &[String],
395) -> Result<(Framed<UnixStream, LengthDelimitedCodec>, SshTunnel), BoxError> {
396    use std::process::Stdio;
397
398    let remote_sock = resolve_remote_sock(ssh_args).await?;
399
400    let local_sock =
401        infinity_protocol::state_dir().join(format!("remote-{}.sock", uuid::Uuid::new_v4()));
402    // Ensure no stale socket
403    let _ = std::fs::remove_file(&local_sock);
404
405    let mut child = tokio::process::Command::new("ssh")
406        .args(ssh_args)
407        .arg("-L")
408        .arg(format!("{}:{}", local_sock.display(), remote_sock))
409        .arg("-N")
410        .stdin(Stdio::null())
411        .stdout(Stdio::null())
412        .stderr(Stdio::null())
413        .kill_on_drop(true)
414        .spawn()
415        .map_err(|e| format!("ssh tunnel spawn failed: {e}"))?;
416
417    // Wait for the local socket to appear (SSH needs a moment to bind it)
418    'outer: for _ in 0..10 {
419        for _ in 0..50 {
420            if local_sock.exists() {
421                break 'outer;
422            } else if let Ok(Some(e)) = child.try_wait() {
423                return Err(format!("SSH child exited while waiting for connection, {e}").into());
424            }
425
426            tokio::time::sleep(std::time::Duration::from_millis(100)).await;
427        }
428
429        tracing::warn!("SSH tunnel not ready, waiting for 5 seconds");
430        tokio::time::sleep(std::time::Duration::from_millis(5000)).await;
431    }
432
433    let stream = UnixStream::connect(&local_sock)
434        .await
435        .map_err(|e| format!("connect to SSH tunnel socket failed: {e}"))?;
436
437    let tunnel = SshTunnel {
438        _child: child,
439        local_sock,
440    };
441
442    Ok((Framed::new(stream, length_delimited_codec()), tunnel))
443}
444
445/// Long-running control worker for a single remote daemon.
446/// Maintains the connection and directly broadcasts prefixed SessionsUpdated.
447async fn control_worker(
448    name: infinity_protocol::RemoteName,
449    ssh_args: Vec<String>,
450    remotes: RemoteMap,
451    broadcast_clients: BroadcastClients,
452) {
453    loop {
454        {
455            let mut map = remotes.lock().expect("bug: mutex poisoned");
456            if let Some(state) = map.get_mut(&name) {
457                state.status = RemoteStatus::Connecting;
458            }
459            drop(map);
460            broadcast(
461                &broadcast_clients,
462                DaemonMessage::RemotesUpdated {
463                    remotes: build_remote_info_list(&remotes),
464                },
465            );
466        }
467
468        match control_worker_inner(&name, &ssh_args, &remotes, &broadcast_clients).await {
469            Ok(()) => {
470                tracing::info!("Remote '{}' control connection closed cleanly", name);
471            }
472            Err(e) => {
473                tracing::warn!("Remote '{}' control connection failed: {e}", name);
474                let mut map = remotes.lock().expect("bug: mutex poisoned");
475                if let Some(state) = map.get_mut(&name) {
476                    state.status = RemoteStatus::Disconnected(e.to_string());
477                    state.sessions.clear();
478                    state.local_sock = None;
479                }
480                drop(map);
481                broadcast(
482                    &broadcast_clients,
483                    DaemonMessage::RemotesUpdated {
484                        remotes: build_remote_info_list(&remotes),
485                    },
486                );
487            }
488        }
489
490        tokio::time::sleep(std::time::Duration::from_secs(5)).await;
491    }
492}
493
494async fn control_worker_inner(
495    name: &infinity_protocol::RemoteName,
496    ssh_args: &[String],
497    remotes: &RemoteMap,
498    broadcast_clients: &BroadcastClients,
499) -> Result<(), BoxError> {
500    let (mut framed, _tunnel) = open_remote_connection(ssh_args).await?;
501
502    tracing::info!("Established remote connection to {name}");
503
504    // Store the tunnel's local socket path so connect_remote_session can reuse it.
505    {
506        let mut map = remotes.lock().expect("bug: mutex poisoned");
507        if let Some(state) = map.get_mut(name) {
508            state.local_sock = Some(_tunnel.local_sock.clone());
509        }
510    }
511
512    // Read Welcome
513    let first = framed
514        .next()
515        .await
516        .ok_or("remote closed before Welcome")?
517        .map_err(|e| format!("read error: {e}"))?;
518    let welcome: DaemonMessage =
519        serde_json::from_slice(&first).map_err(|e| format!("invalid Welcome: {e}"))?;
520
521    let DaemonMessage::Welcome { sessions, .. } = welcome else {
522        return Err("expected Welcome as first message".into());
523    };
524
525    tracing::debug!("Received remote sessions {:?}", &sessions);
526
527    // Update state and broadcast prefixed sessions
528    let prefixed = prefix_sessions(name, sessions.clone());
529    {
530        let mut map = remotes.lock().expect("bug: mutex poisoned");
531        if let Some(state) = map.get_mut(name) {
532            state.status = RemoteStatus::Connected;
533            state.sessions = sessions;
534        }
535    }
536    broadcast(
537        broadcast_clients,
538        DaemonMessage::SessionsUpdated { sessions: prefixed },
539    );
540    broadcast(
541        broadcast_clients,
542        DaemonMessage::RemotesUpdated {
543            remotes: build_remote_info_list(remotes),
544        },
545    );
546    tracing::info!("Remote '{}' connected", name);
547
548    // Listen for SessionsUpdated
549    while let Some(frame) = framed.next().await {
550        let bytes = frame.map_err(|e| format!("read error: {e}"))?;
551        let msg: DaemonMessage =
552            serde_json::from_slice(&bytes).map_err(|e| format!("parse error: {e}"))?;
553
554        if let DaemonMessage::SessionsUpdated { sessions } = msg {
555            let prefixed = prefix_sessions(name, sessions.clone());
556            let mut map = remotes.lock().expect("bug: mutex poisoned");
557            if let Some(state) = map.get_mut(name) {
558                for (id, info) in sessions {
559                    state.sessions.insert(id, info);
560                }
561            }
562            drop(map);
563            broadcast(
564                broadcast_clients,
565                DaemonMessage::SessionsUpdated { sessions: prefixed },
566            );
567        }
568    }
569
570    Ok(())
571}
572
573fn build_remote_info_list(remotes: &RemoteMap) -> Vec<infinity_protocol::RemoteInfo> {
574    let map = remotes.lock().expect("bug: mutex poisoned");
575    map.iter()
576        .map(|(name, state)| infinity_protocol::RemoteInfo {
577            name: name.clone(),
578            status: match &state.status {
579                RemoteStatus::Connecting => "connecting".to_owned(),
580                RemoteStatus::Connected => "connected".to_owned(),
581                RemoteStatus::Disconnected(reason) => format!("disconnected: {reason}"),
582            },
583        })
584        .collect()
585}
586
587fn broadcast(bc: &BroadcastClients, msg: DaemonMessage) {
588    bc.lock()
589        .expect("bug: mutex poisoned")
590        .retain(|tx| tx.send(msg.clone()).is_ok());
591}