1use std::path::{Path, PathBuf};
21
22use serde::{Deserialize, Serialize};
23use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncWrite, AsyncWriteExt, BufReader};
24use tokio::sync::mpsc::UnboundedSender;
25use tokio::sync::{broadcast, oneshot};
26
27use crate::components::AgentStatus;
28use crate::host::{ControlOp, SpawnArgs, WorldEvent};
29use leviath_core::interaction::{InteractionRequest, InteractionResponse};
30
31#[cfg(unix)]
32mod unix;
33#[cfg(unix)]
34pub use unix::{
35 ClientStream, ControlId, ControlListener, ServerStream, bind_control_listener, connect,
36 control_id, control_id_from_str, is_daemon_running,
37};
38
39#[cfg(windows)]
40mod windows;
41#[cfg(windows)]
42pub use windows::{
43 ClientStream, ControlId, ControlListener, ServerStream, bind_control_listener, connect,
44 control_id, control_id_from_str, is_daemon_running,
45};
46
47const AUTH_REQUIRED: &str = "authentication";
53
54#[derive(Clone)]
78pub struct ControlToken(String);
79
80impl std::fmt::Debug for ControlToken {
81 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
84 f.write_str("ControlToken(<redacted>)")
85 }
86}
87
88impl ControlToken {
89 pub fn path(dir: &Path) -> PathBuf {
91 dir.join("control.token")
92 }
93
94 pub fn pid_path(dir: &Path) -> PathBuf {
102 dir.join("daemon.pid")
103 }
104
105 pub fn write_pid(dir: &Path) -> std::io::Result<()> {
107 leviath_sys::write_private(
108 &Self::pid_path(dir),
109 std::process::id().to_string().as_bytes(),
110 )
111 }
112
113 pub fn read_pid(dir: &Path) -> Option<u32> {
115 std::fs::read_to_string(Self::pid_path(dir))
116 .ok()?
117 .trim()
118 .parse()
119 .ok()
120 }
121
122 pub fn create(dir: &Path) -> std::io::Result<Self> {
127 use rand::RngExt as _;
128 let bytes: [u8; 32] = rand::rng().random();
132 let token: String = bytes.iter().map(|b| format!("{b:02x}")).collect();
133
134 std::fs::create_dir_all(dir)?;
135 let _ = leviath_sys::secure_dir_perms(dir);
136 leviath_sys::write_private(&Self::path(dir), token.as_bytes())?;
137 Ok(Self(token))
138 }
139
140 pub fn load(dir: &Path) -> std::io::Result<Self> {
142 let token = std::fs::read_to_string(Self::path(dir))?;
143 Ok(Self(token.trim().to_string()))
144 }
145
146 pub fn matches(&self, presented: &str) -> bool {
152 leviath_core::constant_time_eq(&self.0, presented)
153 }
154
155 pub fn expose(&self) -> &str {
157 &self.0
158 }
159}
160
161#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
164#[serde(tag = "op", rename_all = "snake_case")]
165pub enum ControlRequest {
166 Authenticate {
171 token: String,
173 },
174 Spawn {
176 args: Box<SpawnArgs>,
179 },
180 Status {
182 run_id: String,
184 },
185 Pause {
187 run_id: String,
189 },
190 Resume {
192 run_id: String,
194 },
195 Cancel {
197 run_id: String,
199 },
200 List,
202 Message {
204 agent_id: String,
206 content: String,
208 #[serde(default)]
210 target_region: Option<String>,
211 },
212 ListInteractions,
214 AnswerInteraction {
216 response: InteractionResponse,
218 },
219 CancelInteraction {
221 request_id: String,
223 },
224 Shutdown,
226 Subscribe,
229}
230
231#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
233#[serde(tag = "result", rename_all = "snake_case")]
234pub enum ControlResponse {
235 Spawned {
237 run_id: String,
239 },
240 Status {
242 status: Option<AgentStatus>,
244 },
245 Ok {
247 ok: bool,
249 },
250 List {
252 runs: Vec<(String, AgentStatus)>,
254 },
255 Interactions {
257 interactions: Vec<(String, InteractionRequest)>,
259 },
260 Error {
262 message: String,
264 },
265}
266
267async fn dispatch(req: ControlRequest, op_tx: &UnboundedSender<ControlOp>) -> ControlResponse {
271 match req {
272 ControlRequest::Authenticate { .. } => ControlResponse::Ok { ok: true },
275 ControlRequest::Spawn { args } => {
276 let (reply, rx) = oneshot::channel();
277 let _ = op_tx.send(ControlOp::Spawn { args, reply });
278 match rx.await {
279 Ok(Ok(run_id)) => ControlResponse::Spawned { run_id },
280 Ok(Err(message)) => ControlResponse::Error { message },
281 Err(_) => ControlResponse::Error {
282 message: "daemon is shutting down".to_string(),
283 },
284 }
285 }
286 ControlRequest::Status { run_id } => {
287 let (reply, rx) = oneshot::channel();
288 let _ = op_tx.send(ControlOp::Status { run_id, reply });
289 ControlResponse::Status {
290 status: rx.await.unwrap_or(None),
291 }
292 }
293 ControlRequest::Pause { run_id } => {
294 let (reply, rx) = oneshot::channel();
295 let _ = op_tx.send(ControlOp::Pause { run_id, reply });
296 ControlResponse::Ok {
297 ok: rx.await.unwrap_or(false),
298 }
299 }
300 ControlRequest::Resume { run_id } => {
301 let (reply, rx) = oneshot::channel();
302 let _ = op_tx.send(ControlOp::Resume { run_id, reply });
303 ControlResponse::Ok {
304 ok: rx.await.unwrap_or(false),
305 }
306 }
307 ControlRequest::Cancel { run_id } => {
308 let (reply, rx) = oneshot::channel();
309 let _ = op_tx.send(ControlOp::Cancel { run_id, reply });
310 ControlResponse::Ok {
311 ok: rx.await.unwrap_or(false),
312 }
313 }
314 ControlRequest::List => {
315 let (reply, rx) = oneshot::channel();
316 let _ = op_tx.send(ControlOp::List { reply });
317 ControlResponse::List {
318 runs: rx.await.unwrap_or_default(),
319 }
320 }
321 ControlRequest::Message {
322 agent_id,
323 content,
324 target_region,
325 } => {
326 let (reply, rx) = oneshot::channel();
327 let _ = op_tx.send(ControlOp::Message {
328 agent_id,
329 content,
330 target_region,
331 reply,
332 });
333 ControlResponse::Ok {
334 ok: rx.await.unwrap_or(false),
335 }
336 }
337 ControlRequest::ListInteractions => {
338 let (reply, rx) = oneshot::channel();
339 let _ = op_tx.send(ControlOp::ListInteractions { reply });
340 ControlResponse::Interactions {
341 interactions: rx.await.unwrap_or_default(),
342 }
343 }
344 ControlRequest::AnswerInteraction { response } => {
345 let (reply, rx) = oneshot::channel();
346 let _ = op_tx.send(ControlOp::AnswerInteraction { response, reply });
347 ControlResponse::Ok {
348 ok: rx.await.unwrap_or(false),
349 }
350 }
351 ControlRequest::CancelInteraction { request_id } => {
352 let (reply, rx) = oneshot::channel();
353 let _ = op_tx.send(ControlOp::CancelInteraction { request_id, reply });
354 ControlResponse::Ok {
355 ok: rx.await.unwrap_or(false),
356 }
357 }
358 ControlRequest::Shutdown => {
359 let (reply, rx) = oneshot::channel();
360 let _ = op_tx.send(ControlOp::Shutdown { reply });
361 ControlResponse::Ok {
362 ok: rx.await.unwrap_or(false),
363 }
364 }
365 ControlRequest::Subscribe => ControlResponse::Error {
368 message: "subscribe is a streaming request, not a single-reply op".to_string(),
369 },
370 }
371}
372
373async fn stream_events<W>(
376 write: &mut W,
377 mut rx: broadcast::Receiver<WorldEvent>,
378) -> std::io::Result<()>
379where
380 W: AsyncWrite + Unpin,
381{
382 loop {
383 match rx.recv().await {
384 Ok(event) => {
385 let mut line = serde_json::to_string(&event).expect("WorldEvent serializes");
386 line.push('\n');
387 if write.write_all(line.as_bytes()).await.is_err() {
388 return Ok(()); }
390 }
391 Err(broadcast::error::RecvError::Lagged(_)) => continue,
392 Err(broadcast::error::RecvError::Closed) => return Ok(()),
393 }
394 }
395}
396
397pub async fn handle_connection<S>(
406 stream: S,
407 op_tx: UnboundedSender<ControlOp>,
408 events: broadcast::Sender<WorldEvent>,
409 token: Option<ControlToken>,
410) -> std::io::Result<()>
411where
412 S: AsyncRead + AsyncWrite + Unpin,
413{
414 let (read_half, mut write_half) = tokio::io::split(stream);
415 let mut lines = BufReader::new(read_half).lines();
416 let mut authenticated = token.is_none();
420 while let Some(line) = lines.next_line().await? {
421 if line.trim().is_empty() {
422 continue;
423 }
424
425 if !authenticated {
430 let refused = match serde_json::from_str::<ControlRequest>(&line) {
431 Ok(ControlRequest::Authenticate { token: presented }) => {
432 match token.as_ref().is_some_and(|t| t.matches(&presented)) {
433 true => {
434 authenticated = true;
435 write_line(&mut write_half, &ControlResponse::Ok { ok: true }).await;
436 continue;
437 }
438 false => "authentication failed",
439 }
440 }
441 _ => "authentication required: send an `authenticate` request first",
442 };
443 write_line(
444 &mut write_half,
445 &ControlResponse::Error {
446 message: refused.to_string(),
447 },
448 )
449 .await;
450 return Ok(());
451 }
452
453 let response = match serde_json::from_str::<ControlRequest>(&line) {
454 Ok(ControlRequest::Subscribe) => {
459 let rx = events.subscribe();
460 drop(events);
461 return stream_events(&mut write_half, rx).await;
462 }
463 Ok(ControlRequest::Authenticate { .. }) => ControlResponse::Ok { ok: true },
465 Ok(req) => dispatch(req, &op_tx).await,
466 Err(e) => ControlResponse::Error {
467 message: format!("invalid request: {e}"),
468 },
469 };
470 write_line(&mut write_half, &response).await;
471 }
472 Ok(())
473}
474
475async fn write_line<W>(write_half: &mut W, response: &ControlResponse)
480where
481 W: AsyncWrite + Unpin,
482{
483 let mut out = serde_json::to_string(response).expect("ControlResponse serializes");
485 out.push('\n');
486 let _ = write_half.write_all(out.as_bytes()).await;
487}
488
489pub const DEFAULT_CONTROL_TIMEOUT_SECS: u64 = 30;
495
496pub const SPAWN_CONTROL_TIMEOUT_SECS: u64 = 300;
502
503pub fn request_timeout() -> std::time::Duration {
507 let secs = std::env::var("LEVIATH_CONTROL_TIMEOUT_SECS")
508 .ok()
509 .and_then(|v| v.trim().parse::<u64>().ok())
510 .unwrap_or(DEFAULT_CONTROL_TIMEOUT_SECS);
511 match secs {
512 0 => std::time::Duration::MAX,
513 secs => std::time::Duration::from_secs(secs),
514 }
515}
516
517fn timeout_for(req: &ControlRequest) -> std::time::Duration {
521 let base = request_timeout();
522 match req {
523 ControlRequest::Spawn { .. } if base != std::time::Duration::MAX => {
524 base.max(std::time::Duration::from_secs(SPAWN_CONTROL_TIMEOUT_SECS))
525 }
526 _ => base,
527 }
528}
529
530#[derive(Clone)]
535pub struct ControlClient {
536 id: ControlId,
537 token: Option<ControlToken>,
538 token_dir: Option<PathBuf>,
540}
541
542impl ControlClient {
543 pub fn new(id: impl Into<ControlId>) -> Self {
549 Self {
550 id: id.into(),
551 token: None,
552 token_dir: None,
553 }
554 }
555
556 pub fn with_token(mut self, token: ControlToken) -> Self {
558 self.token = Some(token);
559 self
560 }
561
562 pub fn for_home(id: impl Into<ControlId>, dir: &Path) -> Self {
576 Self {
577 id: id.into(),
578 token: ControlToken::load(dir).ok(),
579 token_dir: Some(dir.to_path_buf()),
580 }
581 }
582
583 fn refused(&self) -> std::io::Error {
585 let detail = match (&self.token, &self.token_dir) {
586 (None, Some(dir)) => format!(
587 "no control token was found at {}. If a daemon is running, it was \
588 started by a different user or before this file existed - restart \
589 it with `lev daemon restart`.",
590 ControlToken::path(dir).display()
591 ),
592 _ => "the daemon refused this client's control token. Restart it with \
593 `lev daemon restart` to issue a fresh one."
594 .to_string(),
595 };
596 std::io::Error::new(std::io::ErrorKind::PermissionDenied, detail)
597 }
598
599 pub async fn request(&self, req: &ControlRequest) -> std::io::Result<ControlResponse> {
603 let deadline = timeout_for(req);
609 tokio::time::timeout(deadline, self.request_uncapped(req))
610 .await
611 .unwrap_or_else(|_| {
612 Err(std::io::Error::new(
613 std::io::ErrorKind::TimedOut,
614 format!("the daemon did not respond within {}s", deadline.as_secs()),
615 ))
616 })
617 }
618
619 async fn request_uncapped(&self, req: &ControlRequest) -> std::io::Result<ControlResponse> {
621 let stream = connect(&self.id).await?;
622 let (read_half, mut write_half) = tokio::io::split(stream);
623 let mut lines = BufReader::new(read_half).lines();
624
625 if let Some(token) = &self.token {
631 let hello = ControlRequest::Authenticate {
632 token: token.expose().to_string(),
633 };
634 let mut line = serde_json::to_string(&hello).expect("ControlRequest serializes");
635 line.push('\n');
636 let _ = write_half.write_all(line.as_bytes()).await;
637
638 match lines.next_line().await.ok().flatten() {
642 Some(resp) => match serde_json::from_str::<ControlResponse>(&resp) {
643 Ok(ControlResponse::Ok { ok: true }) => {}
644 _ => return Err(self.refused()),
645 },
646 None => {
647 return Err(std::io::Error::new(
648 std::io::ErrorKind::UnexpectedEof,
649 "control connection closed during authentication",
650 ));
651 }
652 }
653 }
654
655 let mut line = serde_json::to_string(req).expect("ControlRequest serializes");
656 line.push('\n');
657 let _ = write_half.write_all(line.as_bytes()).await;
660
661 match lines.next_line().await? {
662 Some(resp_line) => {
663 let parsed: ControlResponse = serde_json::from_str(&resp_line)
664 .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
665 match &parsed {
669 ControlResponse::Error { message } if message.starts_with(AUTH_REQUIRED) => {
670 Err(self.refused())
671 }
672 _ => Ok(parsed),
673 }
674 }
675 None => Err(std::io::Error::new(
676 std::io::ErrorKind::UnexpectedEof,
677 "control connection closed before a response",
678 )),
679 }
680 }
681
682 pub async fn spawn(&self, args: SpawnArgs) -> std::io::Result<ControlResponse> {
684 self.request(&ControlRequest::Spawn {
685 args: Box::new(args),
686 })
687 .await
688 }
689
690 pub async fn status(&self, run_id: &str) -> std::io::Result<ControlResponse> {
692 self.request(&ControlRequest::Status {
693 run_id: run_id.to_string(),
694 })
695 .await
696 }
697
698 pub async fn list(&self) -> std::io::Result<ControlResponse> {
700 self.request(&ControlRequest::List).await
701 }
702
703 pub async fn shutdown(&self) -> std::io::Result<ControlResponse> {
705 self.request(&ControlRequest::Shutdown).await
706 }
707
708 pub async fn subscribe(&self) -> std::io::Result<WorldEventStream> {
712 let stream = connect(&self.id).await?;
713 let (read_half, mut write_half) = tokio::io::split(stream);
714 let mut line =
715 serde_json::to_string(&ControlRequest::Subscribe).expect("ControlRequest serializes");
716 line.push('\n');
717 let _ = write_half.write_all(line.as_bytes()).await;
720 Ok(WorldEventStream {
721 lines: BufReader::new(read_half).lines(),
722 _write: write_half,
723 })
724 }
725}
726
727pub struct WorldEventStream {
730 lines: tokio::io::Lines<BufReader<tokio::io::ReadHalf<ClientStream>>>,
731 _write: tokio::io::WriteHalf<ClientStream>,
733}
734
735impl WorldEventStream {
736 pub async fn next(&mut self) -> Option<WorldEvent> {
738 let line = self.lines.next_line().await.ok().flatten()?;
739 serde_json::from_str(&line).ok()
740 }
741}
742
743#[cfg(test)]
744mod tests {
745
746 #[test]
752 fn the_daemon_pid_round_trips_and_a_missing_or_junk_file_is_no_pid() {
753 let dir = tempfile::tempdir().unwrap();
754 assert_eq!(
755 ControlToken::read_pid(dir.path()),
756 None,
757 "no file yet is not a pid"
758 );
759
760 ControlToken::write_pid(dir.path()).unwrap();
761 assert_eq!(
762 ControlToken::read_pid(dir.path()),
763 Some(std::process::id()),
764 "what was written is what comes back"
765 );
766
767 std::fs::write(ControlToken::pid_path(dir.path()), " 4242\n").unwrap();
770 assert_eq!(ControlToken::read_pid(dir.path()), Some(4242));
771 std::fs::write(ControlToken::pid_path(dir.path()), "not-a-pid").unwrap();
772 assert_eq!(ControlToken::read_pid(dir.path()), None);
773 }
774
775 #[test]
778 fn a_token_round_trips_through_its_file_and_is_owner_only() {
779 let dir = tempfile::tempdir().unwrap();
780 let created = ControlToken::create(dir.path()).unwrap();
781 let loaded = ControlToken::load(dir.path()).unwrap();
782
783 assert!(
784 created.matches(loaded.expose()),
785 "the same secret comes back"
786 );
787 assert_eq!(created.expose().len(), 64, "256 bits, hex encoded");
788 let rendered = created.expose().to_string();
789 assert!(
790 rendered.chars().all(|c| c.is_ascii_hexdigit()),
791 "one printable line: {rendered}"
792 );
793
794 #[cfg(unix)]
795 {
796 use std::os::unix::fs::PermissionsExt;
797 let mode = std::fs::metadata(ControlToken::path(dir.path()))
798 .unwrap()
799 .permissions()
800 .mode();
801 assert_eq!(mode & 0o777, 0o600, "the token must be owner-only");
802 }
803 }
804
805 #[test]
808 fn each_token_is_different() {
809 let a = tempfile::tempdir().unwrap();
810 let b = tempfile::tempdir().unwrap();
811 let first = ControlToken::create(a.path()).unwrap();
812 let second = ControlToken::create(b.path()).unwrap();
813 assert!(!first.matches(second.expose()), "tokens must not repeat");
814 }
815
816 #[test]
817 fn a_wrong_or_truncated_token_does_not_match() {
818 let dir = tempfile::tempdir().unwrap();
819 let token = ControlToken::create(dir.path()).unwrap();
820 assert!(!token.matches(""));
821 assert!(!token.matches("deadbeef"));
822 let half: String = token.expose().chars().take(32).collect();
824 assert!(!token.matches(&half));
825 assert!(token.matches(token.expose()));
826 }
827
828 #[test]
831 fn the_token_is_redacted_in_debug_output() {
832 let dir = tempfile::tempdir().unwrap();
833 let token = ControlToken::create(dir.path()).unwrap();
834 let rendered = format!("{token:?}");
835 assert!(!rendered.contains(token.expose()), "{rendered}");
836 assert!(rendered.contains("redacted"), "{rendered}");
837 }
838
839 #[test]
846 fn a_missing_token_still_builds_a_client() {
847 let dir = tempfile::tempdir().unwrap();
848 let client = ControlClient::for_home(control_id(dir.path()), dir.path());
849 let err = client.refused().to_string();
850 assert!(err.contains("no control token was found"), "{err}");
851 assert!(err.contains("lev daemon restart"), "{err}");
852 }
853
854 #[test]
857 fn a_rejected_token_reads_differently_from_a_missing_one() {
858 let dir = tempfile::tempdir().unwrap();
859 let _token = ControlToken::create(dir.path()).unwrap();
860 let client = ControlClient::for_home(control_id(dir.path()), dir.path());
861 let err = client.refused().to_string();
862 assert!(err.contains("refused this client's control token"), "{err}");
863 assert!(!err.contains("no control token was found"), "{err}");
864 }
865 use super::*;
866 use tokio::sync::mpsc;
867
868 fn no_events() -> broadcast::Sender<WorldEvent> {
870 broadcast::channel(16).0
871 }
872
873 fn spawn_fake_host(mut rx: mpsc::UnboundedReceiver<ControlOp>) {
875 tokio::spawn(async move {
876 while let Some(op) = rx.recv().await {
877 match op {
878 ControlOp::Spawn { args, reply } => {
879 let result = if args.run_id == "FAIL" {
881 Err("bad blueprint".to_string())
882 } else {
883 Ok(args.run_id)
884 };
885 let _ = reply.send(result);
886 }
887 ControlOp::Status { reply, .. } => {
888 let _ = reply.send(Some(AgentStatus::Active));
889 }
890 ControlOp::Pause { reply, .. }
891 | ControlOp::Resume { reply, .. }
892 | ControlOp::Cancel { reply, .. } => {
893 let _ = reply.send(true);
894 }
895 ControlOp::Message { reply, .. }
896 | ControlOp::AnswerInteraction { reply, .. }
897 | ControlOp::CancelInteraction { reply, .. }
898 | ControlOp::Shutdown { reply } => {
899 let _ = reply.send(true);
900 }
901 ControlOp::List { reply } => {
902 let _ = reply.send(vec![("run-a".to_string(), AgentStatus::Active)]);
903 }
904 ControlOp::ListInteractions { reply } => {
905 let _ = reply.send(vec![]);
906 }
907 }
908 }
909 });
910 }
911
912 fn test_listener() -> (ControlListener, ControlId, tempfile::TempDir) {
915 let dir = tempfile::tempdir().unwrap();
916 let id = control_id(dir.path());
917 let listener = bind_control_listener(&id).unwrap();
918 (listener, id, dir)
919 }
920
921 async fn round_trip(req: &ControlRequest) -> ControlResponse {
922 let (op_tx, op_rx) = mpsc::unbounded_channel();
923 spawn_fake_host(op_rx);
924 let (mut listener, id, _dir) = test_listener();
925 tokio::spawn(async move {
926 let stream = listener
929 .accept()
930 .await
931 .expect("accept succeeds")
932 .expect("our own connection is admitted");
933 let _ = handle_connection(stream, op_tx, no_events(), None).await;
934 });
935
936 let stream = connect(&id).await.unwrap();
937 let (read_half, mut write_half) = tokio::io::split(stream);
938 let mut line = serde_json::to_string(req).unwrap();
939 line.push('\n');
940 write_half.write_all(line.as_bytes()).await.unwrap();
941
942 let mut lines = BufReader::new(read_half).lines();
943 let resp_line = lines.next_line().await.unwrap().unwrap();
944 serde_json::from_str(&resp_line).unwrap()
945 }
946
947 #[tokio::test]
948 async fn status_request_round_trips() {
949 let resp = round_trip(&ControlRequest::Status {
950 run_id: "run-a".to_string(),
951 })
952 .await;
953 assert_eq!(
954 resp,
955 ControlResponse::Status {
956 status: Some(AgentStatus::Active)
957 }
958 );
959 }
960
961 #[tokio::test]
962 async fn control_ops_round_trip() {
963 for req in [
964 ControlRequest::Pause {
965 run_id: "r".to_string(),
966 },
967 ControlRequest::Resume {
968 run_id: "r".to_string(),
969 },
970 ControlRequest::Cancel {
971 run_id: "r".to_string(),
972 },
973 ControlRequest::Message {
974 agent_id: "a".to_string(),
975 content: "hi".to_string(),
976 target_region: None,
977 },
978 ControlRequest::AnswerInteraction {
979 response: InteractionResponse::text("q1", "yes"),
980 },
981 ControlRequest::CancelInteraction {
982 request_id: "q1".to_string(),
983 },
984 ] {
985 assert_eq!(round_trip(&req).await, ControlResponse::Ok { ok: true });
986 }
987 }
988
989 #[tokio::test]
990 async fn spawn_request_round_trips() {
991 let resp = round_trip(&ControlRequest::Spawn {
992 args: Box::new(SpawnArgs {
993 run_id: "run-9".to_string(),
994 blueprint_path: "/agents/x".to_string(),
995 task: "do it".to_string(),
996 regions: Default::default(),
997 model: None,
998 workdir: "/w".to_string(),
999 metadata: Default::default(),
1000 callback_url: None,
1001 callback_secret: None,
1002 yolo: false,
1003 no_seed_commands: false,
1004 allow: Vec::new(),
1005 max_depth: None,
1006 parent_run_id: None,
1007 }),
1008 })
1009 .await;
1010 assert_eq!(
1011 resp,
1012 ControlResponse::Spawned {
1013 run_id: "run-9".to_string()
1014 }
1015 );
1016 }
1017
1018 #[tokio::test]
1019 async fn spawn_error_from_host_becomes_error_response() {
1020 let resp = round_trip(&ControlRequest::Spawn {
1021 args: Box::new(SpawnArgs {
1022 run_id: "FAIL".to_string(),
1023 ..Default::default()
1024 }),
1025 })
1026 .await;
1027 assert_eq!(
1028 std::mem::discriminant(&resp),
1029 std::mem::discriminant(&ControlResponse::Error {
1030 message: String::new()
1031 })
1032 );
1033 }
1034
1035 #[tokio::test]
1036 async fn list_interactions_round_trips() {
1037 let resp = round_trip(&ControlRequest::ListInteractions).await;
1038 assert_eq!(
1039 resp,
1040 ControlResponse::Interactions {
1041 interactions: vec![]
1042 }
1043 );
1044 }
1045
1046 #[tokio::test]
1047 async fn list_request_round_trips() {
1048 let resp = round_trip(&ControlRequest::List).await;
1049 assert_eq!(
1050 resp,
1051 ControlResponse::List {
1052 runs: vec![("run-a".to_string(), AgentStatus::Active)]
1053 }
1054 );
1055 }
1056
1057 #[tokio::test]
1058 async fn shutdown_request_round_trips() {
1059 assert_eq!(
1060 round_trip(&ControlRequest::Shutdown).await,
1061 ControlResponse::Ok { ok: true }
1062 );
1063 }
1064
1065 fn completed(run_id: &str) -> WorldEvent {
1066 WorldEvent::Completed {
1067 run_id: run_id.to_string(),
1068 agent_id: "a".to_string(),
1069 status: "complete".to_string(),
1070 }
1071 }
1072
1073 #[tokio::test]
1074 async fn dispatch_rejects_subscribe_as_a_single_reply_op() {
1075 let (op_tx, _rx) = mpsc::unbounded_channel();
1076 let resp = dispatch(ControlRequest::Subscribe, &op_tx).await;
1077 assert_eq!(
1078 std::mem::discriminant(&resp),
1079 std::mem::discriminant(&ControlResponse::Error {
1080 message: String::new()
1081 })
1082 );
1083 }
1084
1085 #[tokio::test]
1086 async fn stream_events_skips_lagged_writes_ok_and_stops_on_closed() {
1087 use tokio::io::AsyncReadExt;
1088 let (tx, rx) = broadcast::channel::<WorldEvent>(1);
1089 tx.send(completed("first")).unwrap();
1091 tx.send(completed("second")).unwrap();
1092 tx.send(completed("third")).unwrap();
1093 drop(tx); let (mut w, mut r) = tokio::io::duplex(4096);
1096 let server = tokio::spawn(async move { stream_events(&mut w, rx).await });
1097 let mut buf = String::new();
1098 r.read_to_string(&mut buf).await.unwrap();
1099 server.await.unwrap().unwrap();
1100 assert!(buf.contains("third"));
1102 assert!(!buf.contains("first"));
1103 }
1104
1105 #[tokio::test]
1106 async fn stream_events_returns_when_the_client_hangs_up() {
1107 let (tx, rx) = broadcast::channel::<WorldEvent>(4);
1108 tx.send(completed("x")).unwrap();
1109 let (mut w, r) = tokio::io::duplex(64);
1110 drop(r); stream_events(&mut w, rx).await.unwrap();
1112 drop(tx);
1113 }
1114
1115 #[test]
1119 fn creating_a_token_in_an_unwritable_place_is_an_error() {
1120 let dir = tempfile::tempdir().unwrap();
1121 let blocker = dir.path().join("blocker");
1123 std::fs::write(&blocker, b"x").unwrap();
1124 assert!(
1125 ControlToken::create(&blocker.join("nested")).is_err(),
1126 "a directory that cannot be created is an error"
1127 );
1128
1129 let occupied = dir.path().join("occupied");
1132 std::fs::create_dir_all(ControlToken::path(&occupied)).unwrap();
1133 assert!(
1134 ControlToken::create(&occupied).is_err(),
1135 "a token file that cannot be written is an error"
1136 );
1137 }
1138
1139 #[tokio::test]
1143 async fn dispatching_authenticate_is_inert() {
1144 let (op_tx, _op_rx) = mpsc::unbounded_channel();
1145 let response = dispatch(
1146 ControlRequest::Authenticate {
1147 token: "irrelevant".to_string(),
1148 },
1149 &op_tx,
1150 )
1151 .await;
1152 assert!(matches!(response, ControlResponse::Ok { ok: true }));
1153 }
1154
1155 #[tokio::test]
1158 async fn re_authenticating_on_an_open_connection_is_accepted() {
1159 let (events, _r) = broadcast::channel::<WorldEvent>(16);
1160 let (op_tx, op_rx) = mpsc::unbounded_channel();
1161 spawn_fake_host(op_rx);
1162 let (mut listener, id, dir) = test_listener();
1163 let token = ControlToken::create(dir.path()).unwrap();
1164
1165 let server_token = token.clone();
1166 let server = tokio::spawn(async move {
1167 let stream = listener.accept().await.unwrap().unwrap();
1168 let _ = handle_connection(stream, op_tx, events, Some(server_token)).await;
1169 });
1170
1171 let stream = connect(&id).await.unwrap();
1172 let (read_half, mut write_half) = tokio::io::split(stream);
1173 let mut lines = BufReader::new(read_half).lines();
1174 let hello = serde_json::to_string(&ControlRequest::Authenticate {
1175 token: token.expose().to_string(),
1176 })
1177 .unwrap();
1178 for _ in 0..2 {
1179 write_half
1180 .write_all(format!("{hello}\n").as_bytes())
1181 .await
1182 .unwrap();
1183 let resp: ControlResponse =
1184 serde_json::from_str(&lines.next_line().await.unwrap().unwrap()).unwrap();
1185 let rendered = format!("{resp:?}");
1186 assert!(rendered.starts_with("Ok"), "{rendered}");
1187 }
1188
1189 drop(write_half);
1191 drop(lines);
1192 server.await.unwrap();
1193 }
1194
1195 #[tokio::test]
1198 async fn a_connection_closed_during_authentication_is_reported() {
1199 let (mut listener, id, dir) = test_listener();
1200 let token = ControlToken::create(dir.path()).unwrap();
1201 tokio::spawn(async move {
1202 let _ = listener.accept().await;
1204 });
1205
1206 let err = ControlClient::new(id)
1207 .with_token(token)
1208 .list()
1209 .await
1210 .expect_err("a hang-up during authentication is an error");
1211 assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
1212 assert!(err.to_string().contains("during authentication"), "{err}");
1213 }
1214
1215 #[tokio::test]
1219 async fn an_unauthenticated_caller_is_refused_and_disconnected() {
1220 let (events, _r) = broadcast::channel::<WorldEvent>(16);
1221 let (op_tx, op_rx) = mpsc::unbounded_channel();
1222 spawn_fake_host(op_rx);
1223 let (mut listener, id, dir) = test_listener();
1224 let token = ControlToken::create(dir.path()).unwrap();
1225
1226 let server_token = token.clone();
1227 let server = tokio::spawn(async move {
1228 let stream = listener.accept().await.unwrap().unwrap();
1229 let _ = handle_connection(stream, op_tx, events, Some(server_token)).await;
1230 });
1231
1232 let err = ControlClient::new(id)
1237 .list()
1238 .await
1239 .expect_err("an unauthenticated List must not be served");
1240 assert_eq!(err.kind(), std::io::ErrorKind::PermissionDenied);
1241 assert!(err.to_string().contains("token"), "{err}");
1242 server.await.unwrap();
1244 }
1245
1246 #[tokio::test]
1248 async fn a_client_presenting_the_wrong_token_is_refused() {
1249 let (events, _r) = broadcast::channel::<WorldEvent>(16);
1250 let (op_tx, op_rx) = mpsc::unbounded_channel();
1251 spawn_fake_host(op_rx);
1252 let (mut listener, id, dir) = test_listener();
1253 let real = ControlToken::create(dir.path()).unwrap();
1254
1255 tokio::spawn(async move {
1256 let stream = listener.accept().await.unwrap().unwrap();
1257 let _ = handle_connection(stream, op_tx, events, Some(real)).await;
1258 });
1259
1260 let other_dir = tempfile::tempdir().unwrap();
1262 let wrong = ControlToken::create(other_dir.path()).unwrap();
1263 let err = ControlClient::new(id)
1264 .with_token(wrong)
1265 .list()
1266 .await
1267 .expect_err("a wrong token is refused");
1268 assert_eq!(err.kind(), std::io::ErrorKind::PermissionDenied);
1269 assert!(err.to_string().contains("refused"), "{err}");
1270 }
1271
1272 #[tokio::test]
1275 async fn a_client_presenting_the_right_token_is_served() {
1276 let (events, _r) = broadcast::channel::<WorldEvent>(16);
1277 let (op_tx, op_rx) = mpsc::unbounded_channel();
1278 spawn_fake_host(op_rx);
1279 let (mut listener, id, dir) = test_listener();
1280 let token = ControlToken::create(dir.path()).unwrap();
1281
1282 let server_token = token.clone();
1283 let server = tokio::spawn(async move {
1284 let stream = listener.accept().await.unwrap().unwrap();
1285 let _ = handle_connection(stream, op_tx, events, Some(server_token)).await;
1286 });
1287
1288 let client = ControlClient::for_home(id, dir.path());
1290 let response = client
1291 .list()
1292 .await
1293 .expect("an authenticated List is served");
1294 let rendered = format!("{response:?}");
1295 assert!(
1296 rendered.starts_with("List"),
1297 "expected a run list: {rendered}"
1298 );
1299 server.await.unwrap();
1301 }
1302
1303 #[tokio::test]
1304 async fn subscribe_streams_events_to_the_client() {
1305 let (events, _r) = broadcast::channel::<WorldEvent>(16);
1306 let (op_tx, op_rx) = mpsc::unbounded_channel();
1307 spawn_fake_host(op_rx);
1308 let (mut listener, id, _dir) = test_listener();
1309 let server_events = events.clone();
1310 let server = tokio::spawn(async move {
1311 let stream = listener
1314 .accept()
1315 .await
1316 .expect("accept succeeds")
1317 .expect("our own connection is admitted");
1318 let _ = handle_connection(stream, op_tx, server_events, None).await;
1319 });
1320
1321 let mut stream = ControlClient::new(id).subscribe().await.unwrap();
1322 let received = loop {
1324 events.send(completed("run-1")).unwrap();
1325 tokio::select! {
1326 e = stream.next() => break e,
1327 _ = tokio::time::sleep(std::time::Duration::from_millis(5)) => {}
1328 }
1329 };
1330 let received = received.expect("an event should have streamed to the client");
1331 assert_eq!(
1332 std::mem::discriminant(&received),
1333 std::mem::discriminant(&completed("x"))
1334 );
1335 drop(events);
1337 server.await.unwrap();
1338 }
1339
1340 #[tokio::test]
1341 async fn subscribe_errors_when_daemon_absent() {
1342 let dir = tempfile::tempdir().unwrap();
1343 let client = ControlClient::new(control_id(&dir.path().join("no-daemon")));
1344 assert!(client.subscribe().await.is_err());
1345 }
1346
1347 #[tokio::test]
1348 async fn subscribe_stream_ends_when_the_daemon_closes() {
1349 let (events, _r) = broadcast::channel::<WorldEvent>(16);
1350 let (op_tx, op_rx) = mpsc::unbounded_channel();
1351 spawn_fake_host(op_rx);
1352 let (mut listener, id, _dir) = test_listener();
1353 let server_events = events.clone();
1354 tokio::spawn(async move {
1355 let stream = listener
1358 .accept()
1359 .await
1360 .expect("accept succeeds")
1361 .expect("our own connection is admitted");
1362 let _ = handle_connection(stream, op_tx, server_events, None).await;
1363 });
1364
1365 let mut stream = ControlClient::new(id).subscribe().await.unwrap();
1366 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1369 drop(events);
1370 assert!(stream.next().await.is_none());
1371 }
1372
1373 async fn connected_pair() -> (ClientStream, ServerStream, tempfile::TempDir) {
1376 let (mut listener, id, dir) = test_listener();
1377 let (client, server) = tokio::join!(connect(&id), listener.accept());
1378 let server = server
1379 .expect("accept succeeds")
1380 .expect("our own connection is admitted");
1381 (client.unwrap(), server, dir)
1382 }
1383
1384 #[tokio::test]
1385 async fn malformed_request_gets_error_and_connection_continues() {
1386 let (op_tx, op_rx) = mpsc::unbounded_channel();
1387 spawn_fake_host(op_rx);
1388 let (client, server, _dir) = connected_pair().await;
1389 let handle =
1390 tokio::spawn(async move { handle_connection(server, op_tx, no_events(), None).await });
1391
1392 let (read_half, mut write_half) = tokio::io::split(client);
1393 write_half.write_all(b"\nnot json\n").await.unwrap();
1395 let mut lines = BufReader::new(read_half).lines();
1396 let err_line = lines.next_line().await.unwrap().unwrap();
1397 let resp: ControlResponse = serde_json::from_str(&err_line).unwrap();
1398 assert_eq!(
1399 std::mem::discriminant(&resp),
1400 std::mem::discriminant(&ControlResponse::Error {
1401 message: String::new()
1402 })
1403 );
1404
1405 let mut valid = serde_json::to_string(&ControlRequest::List).unwrap();
1407 valid.push('\n');
1408 write_half.write_all(valid.as_bytes()).await.unwrap();
1409 let ok_line = lines.next_line().await.unwrap().unwrap();
1410 let ok: ControlResponse = serde_json::from_str(&ok_line).unwrap();
1411 assert_eq!(
1412 std::mem::discriminant(&ok),
1413 std::mem::discriminant(&ControlResponse::List { runs: vec![] })
1414 );
1415
1416 drop(write_half);
1418 drop(lines);
1419 handle.await.unwrap().unwrap();
1420 }
1421
1422 #[tokio::test]
1423 async fn invalid_utf8_line_ends_connection_with_error() {
1424 let (op_tx, op_rx) = mpsc::unbounded_channel();
1425 spawn_fake_host(op_rx);
1426 let (client, server, _dir) = connected_pair().await;
1427 let handle =
1428 tokio::spawn(async move { handle_connection(server, op_tx, no_events(), None).await });
1429
1430 let (_read_half, mut write_half) = tokio::io::split(client);
1431 write_half.write_all(&[0xff, 0xfe, b'\n']).await.unwrap();
1434
1435 let result = handle.await.unwrap();
1436 assert!(result.is_err());
1437 }
1438
1439 #[tokio::test]
1440 async fn client_round_trips_status_and_list() {
1441 let (op_tx, op_rx) = mpsc::unbounded_channel();
1442 spawn_fake_host(op_rx);
1443 let (mut listener, id, _dir) = test_listener();
1444 tokio::spawn(async move {
1445 for _ in 0..4 {
1446 let stream = listener
1449 .accept()
1450 .await
1451 .expect("accept succeeds")
1452 .expect("our own connection is admitted");
1453 let op_tx = op_tx.clone();
1454 tokio::spawn(async move {
1455 let _ = handle_connection(stream, op_tx, no_events(), None).await;
1456 });
1457 }
1458 });
1459 let client = ControlClient::new(id);
1460
1461 let spawned = client
1462 .spawn(SpawnArgs {
1463 run_id: "r-c".to_string(),
1464 ..Default::default()
1465 })
1466 .await
1467 .unwrap();
1468 assert_eq!(
1469 spawned,
1470 ControlResponse::Spawned {
1471 run_id: "r-c".to_string()
1472 }
1473 );
1474
1475 let status = client.status("run-a").await.unwrap();
1476 assert_eq!(
1477 status,
1478 ControlResponse::Status {
1479 status: Some(AgentStatus::Active)
1480 }
1481 );
1482 let list = client.list().await.unwrap();
1483 assert_eq!(
1484 std::mem::discriminant(&list),
1485 std::mem::discriminant(&ControlResponse::List { runs: vec![] })
1486 );
1487 assert_eq!(
1488 client.shutdown().await.unwrap(),
1489 ControlResponse::Ok { ok: true }
1490 );
1491 }
1492
1493 #[tokio::test]
1494 async fn client_errors_when_daemon_absent() {
1495 let dir = tempfile::tempdir().unwrap();
1496 let id = control_id(&dir.path().join("no-daemon-here"));
1498 assert!(ControlClient::new(id).list().await.is_err());
1499 }
1500
1501 async fn raw_server(bytes: &'static [u8]) -> (ControlId, tempfile::TempDir) {
1504 let (mut listener, id, dir) = test_listener();
1505 tokio::spawn(async move {
1506 let stream = listener
1509 .accept()
1510 .await
1511 .expect("accept succeeds")
1512 .expect("our own connection is admitted");
1513 let (_r, mut w) = tokio::io::split(stream);
1514 let _ = w.write_all(bytes).await;
1515 });
1516 (id, dir)
1517 }
1518
1519 #[tokio::test]
1520 async fn client_errors_on_unparseable_response() {
1521 let (id, _dir) = raw_server(b"not json\n").await;
1523 let err = ControlClient::new(id).list().await.unwrap_err();
1524 assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
1525 }
1526
1527 #[tokio::test]
1528 async fn client_errors_on_invalid_utf8_response() {
1529 let (id, _dir) = raw_server(&[0xff, 0xfe, b'\n']).await;
1531 assert!(ControlClient::new(id).list().await.is_err());
1532 }
1533
1534 #[tokio::test]
1535 async fn client_errors_on_closed_connection_without_reply() {
1536 let (mut listener, id, _dir) = test_listener();
1538 tokio::spawn(async move {
1539 let stream = listener
1542 .accept()
1543 .await
1544 .expect("accept succeeds")
1545 .expect("our own connection is admitted");
1546 let (read_half, _write_half) = tokio::io::split(stream);
1549 let mut lines = BufReader::new(read_half).lines();
1550 let _ = lines.next_line().await;
1551 });
1552
1553 let err = ControlClient::new(id).list().await.unwrap_err();
1554 assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
1555 }
1556
1557 #[tokio::test]
1562 async fn client_times_out_on_a_daemon_that_never_answers() {
1563 let (mut listener, id, _dir) = test_listener();
1564 let (tx, rx) = oneshot::channel();
1570 tokio::spawn(async move {
1571 let stream = listener
1574 .accept()
1575 .await
1576 .expect("accept succeeds")
1577 .expect("our own connection is admitted");
1578 let (read_half, write_half) = tokio::io::split(stream);
1579 let mut lines = BufReader::new(read_half).lines();
1580 let _ = lines.next_line().await;
1581 let _ = tx.send((lines, write_half));
1582 });
1583
1584 let err = temp_env::async_with_vars([("LEVIATH_CONTROL_TIMEOUT_SECS", Some("1"))], async {
1585 ControlClient::new(id).list().await.unwrap_err()
1586 })
1587 .await;
1588 assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
1589 assert!(err.to_string().contains("did not respond"), "got: {err}");
1590 drop(rx);
1592 }
1593
1594 #[test]
1595 fn request_timeout_honors_the_override_and_falls_back() {
1596 temp_env::with_var("LEVIATH_CONTROL_TIMEOUT_SECS", Some("7"), || {
1597 assert_eq!(request_timeout(), std::time::Duration::from_secs(7));
1598 });
1599 temp_env::with_var("LEVIATH_CONTROL_TIMEOUT_SECS", Some("0"), || {
1602 assert_eq!(request_timeout(), std::time::Duration::MAX);
1603 });
1604 temp_env::with_var("LEVIATH_CONTROL_TIMEOUT_SECS", Some("soon"), || {
1606 assert_eq!(
1607 request_timeout(),
1608 std::time::Duration::from_secs(DEFAULT_CONTROL_TIMEOUT_SECS)
1609 );
1610 });
1611 temp_env::with_var_unset("LEVIATH_CONTROL_TIMEOUT_SECS", || {
1612 assert_eq!(
1613 request_timeout(),
1614 std::time::Duration::from_secs(DEFAULT_CONTROL_TIMEOUT_SECS)
1615 );
1616 });
1617 }
1618
1619 #[test]
1623 fn spawn_gets_a_longer_deadline_than_other_ops() {
1624 let spawn = ControlRequest::Spawn {
1625 args: Box::new(SpawnArgs::default()),
1626 };
1627 let cancel = ControlRequest::Cancel {
1628 run_id: "r".to_string(),
1629 };
1630 temp_env::with_var_unset("LEVIATH_CONTROL_TIMEOUT_SECS", || {
1631 assert_eq!(
1632 timeout_for(&spawn),
1633 std::time::Duration::from_secs(SPAWN_CONTROL_TIMEOUT_SECS)
1634 );
1635 assert_eq!(
1636 timeout_for(&cancel),
1637 std::time::Duration::from_secs(DEFAULT_CONTROL_TIMEOUT_SECS)
1638 );
1639 });
1640 temp_env::with_var("LEVIATH_CONTROL_TIMEOUT_SECS", Some("900"), || {
1642 assert_eq!(timeout_for(&spawn), std::time::Duration::from_secs(900));
1643 assert_eq!(timeout_for(&cancel), std::time::Duration::from_secs(900));
1644 });
1645 temp_env::with_var("LEVIATH_CONTROL_TIMEOUT_SECS", Some("0"), || {
1647 assert_eq!(timeout_for(&spawn), std::time::Duration::MAX);
1648 });
1649 }
1650
1651 #[tokio::test]
1652 async fn bind_rejects_when_daemon_already_running() {
1653 let (_live, id, _dir) = test_listener(); let err = bind_control_listener(&id).unwrap_err();
1655 assert_eq!(err.kind(), std::io::ErrorKind::AddrInUse);
1656 }
1657
1658 #[tokio::test]
1659 async fn is_daemon_running_reflects_a_live_listener() {
1660 let dir = tempfile::tempdir().unwrap();
1661 let id = control_id(dir.path());
1662 assert!(!is_daemon_running(&id)); let _live = bind_control_listener(&id).unwrap();
1664 assert!(is_daemon_running(&id)); }
1666
1667 #[tokio::test]
1668 async fn dispatch_returns_neutral_when_host_gone() {
1669 let (op_tx, op_rx) = mpsc::unbounded_channel();
1672 drop(op_rx);
1673 assert_eq!(
1674 dispatch(
1675 ControlRequest::Status {
1676 run_id: "r".to_string()
1677 },
1678 &op_tx
1679 )
1680 .await,
1681 ControlResponse::Status { status: None }
1682 );
1683 assert_eq!(
1684 dispatch(
1685 ControlRequest::Cancel {
1686 run_id: "r".to_string()
1687 },
1688 &op_tx
1689 )
1690 .await,
1691 ControlResponse::Ok { ok: false }
1692 );
1693 assert_eq!(
1694 dispatch(ControlRequest::List, &op_tx).await,
1695 ControlResponse::List { runs: vec![] }
1696 );
1697 assert_eq!(
1698 dispatch(ControlRequest::ListInteractions, &op_tx).await,
1699 ControlResponse::Interactions {
1700 interactions: vec![]
1701 }
1702 );
1703 assert_eq!(
1704 std::mem::discriminant(
1705 &dispatch(
1706 ControlRequest::Spawn {
1707 args: Box::new(SpawnArgs::default())
1708 },
1709 &op_tx
1710 )
1711 .await
1712 ),
1713 std::mem::discriminant(&ControlResponse::Error {
1714 message: String::new()
1715 })
1716 );
1717 }
1718}