use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use anyhow::Context;
use tokio::time::Instant;
use serde_json::Value;
use tokio::sync::broadcast;
use super::{Daemon, RELEASED, is_news};
use crate::agent::{self, codex_remote::Remote};
use crate::ipc::Agent;
use crate::store::{CodexHost, PendingPush};
const GATHER: Duration = Duration::from_secs(1);
const PROBE: Duration = Duration::from_secs(5);
const TIMEOUT: Duration = Duration::from_secs(20);
const RETRY: Duration = Duration::from_secs(30);
const RETRY_MAX: Duration = Duration::from_secs(600);
const HOST_TTL: Duration = Duration::from_secs(60);
const HOST_CAPACITY: usize = 128;
#[derive(Clone, Copy, PartialEq, Debug, serde::Serialize)]
#[serde(rename_all = "lowercase")]
pub(super) enum Reach {
Push,
Doorbell,
Wait,
}
impl Daemon {
pub(super) async fn register_host(
&self,
caller: &str,
env: &[(String, String)],
) -> (Reach, Option<&'static str>) {
let Some(thread) = caller.strip_prefix("codex:") else {
return (Reach::Wait, None);
};
let _delivering = self.delivery(caller).lock_owned().await;
let configured = self.opts.agents.get(Agent::Codex).path.as_deref();
let Ok(exe) = agent::resolve_exe(Agent::Codex, configured, env) else {
return (Reach::Wait, Some("Codex was not found"));
};
let codex_home = env
.iter()
.find(|(k, _)| k == "CODEX_HOME")
.map(|(_, v)| PathBuf::from(v));
let mut host = CodexHost {
exe,
codex_home,
thread: thread.to_string(),
doorbell: false,
};
if let Some(reach) = self.cached_host(caller, &host) {
return (reach, None);
}
let (reach, why) = self.probe_host(caller, &host).await;
if why.is_some() {
return (reach, why);
}
host.doorbell = reach == Reach::Doorbell;
let mut st = self.lock();
st.ledger.codex_hosts.insert(caller.to_string(), host);
Self::cache_host_check(&mut st, caller);
if let Err(e) = st.ledger.save(&self.opts.home) {
self.log(&format!("saving the ledger: {e:#}"));
}
(reach, None)
}
async fn probe_host(&self, caller: &str, host: &CodexHost) -> (Reach, Option<&'static str>) {
let home = host.codex_home.as_deref();
let shared = self
.probe(caller, "is not on the shared Codex daemon", async {
Remote::connect(&host.exe, home)
.await?
.is_open(&host.thread)
.await
})
.await;
let reach = if shared == Some(true) {
Reach::Push
} else {
let queued = self
.probe(caller, "has no Codex queue", async {
Remote::private(&host.exe, home)
.await?
.serves_queue(&host.thread)
.await
})
.await;
let why = match (shared, queued) {
(_, Some(true)) => None,
(Some(false), Some(false)) => Some(
"it is neither on the shared Codex daemon nor open in the Codex app or IDE extension",
),
_ => Some("Codex did not answer within 5 seconds"),
};
if why.is_some() {
return (Reach::Wait, why);
}
Reach::Doorbell
};
(reach, None)
}
fn cache_host_check(st: &mut super::State, caller: &str) {
st.host_checked.retain(|_, at| at.elapsed() < HOST_TTL);
if st.host_checked.len() >= HOST_CAPACITY {
st.host_checked.clear();
}
st.host_checked
.insert(caller.to_string(), std::time::Instant::now());
}
fn cached_host(&self, caller: &str, candidate: &CodexHost) -> Option<Reach> {
let st = self.lock();
let at = st.host_checked.get(caller)?;
let host = st.ledger.codex_hosts.get(caller)?;
if at.elapsed() >= HOST_TTL
|| host.exe != candidate.exe
|| host.codex_home != candidate.codex_home
|| host.thread != candidate.thread
{
return None;
}
Some(if host.doorbell {
Reach::Doorbell
} else {
Reach::Push
})
}
async fn probe(
&self,
caller: &str,
failure: &str,
check: impl Future<Output = anyhow::Result<bool>>,
) -> Option<bool> {
match tokio::time::timeout(PROBE, check).await {
Ok(Ok(yes)) => Some(yes),
Ok(Err(e)) => {
self.log(&format!("{caller} {failure}: {e:#}"));
Some(false)
}
Err(_) => {
self.log(&format!("{caller} {failure}: Codex did not answer"));
None
}
}
}
pub(super) async fn pusher(self: Arc<Self>) {
let mut rx = self.events.subscribe();
self.push_all();
loop {
let e = match rx.recv().await {
Ok(e) => e,
Err(broadcast::error::RecvError::Lagged(_)) => {
self.push_all();
continue;
}
Err(broadcast::error::RecvError::Closed) => return,
};
if e.kind == RELEASED {
continue;
}
let target = {
let st = self.lock();
if e.kind == "approval_request" {
e.data
.get("approver")
.and_then(Value::as_str)
.map(str::to_string)
} else {
st.tasks.get(&e.task).map(|t| t.task.parent.clone())
}
};
if let Some(target) = target
&& is_news(&e, &target)
{
self.schedule_push(target, GATHER);
}
}
}
fn push_all(self: &Arc<Self>) {
let hosts: Vec<String> = self.lock().ledger.codex_hosts.keys().cloned().collect();
for host in hosts {
self.schedule_push(host, GATHER);
}
}
fn schedule_push(self: &Arc<Self>, caller: String, delay: Duration) {
let due = Instant::now() + delay;
{
let mut st = self.lock();
if !st.ledger.codex_hosts.contains_key(&caller)
|| st.pushing.get(&caller).is_some_and(|d| *d <= due)
{
return;
}
st.pushing.insert(caller.clone(), due);
}
let d = self.clone();
tokio::spawn(async move {
tokio::time::sleep_until(due).await;
{
let mut st = d.lock();
if st.pushing.get(&caller) != Some(&due) {
return;
}
st.pushing.remove(&caller);
}
d.push(&caller).await;
});
}
fn clear_pending_push(&self, caller: &str) -> anyhow::Result<()> {
let mut st = self.lock();
let mut ledger = st.ledger.clone();
ledger.pending_pushes.remove(caller);
ledger.save(&self.opts.home)?;
st.ledger = ledger;
Ok(())
}
async fn reconcile_push(
&self,
caller: &str,
thread: &str,
remote: &mut Remote,
) -> anyhow::Result<()> {
let pending = self.lock().ledger.pending_pushes.get(caller).cloned();
let Some(pending) = pending else {
return Ok(());
};
let delivered =
tokio::time::timeout(PROBE, remote.contains_message(thread, &pending.marker))
.await
.context("push read-back timed out; retry later")?
.context("push read-back failed; retry later")?;
self.settle_push(caller, pending, delivered)
}
fn settle_push(
&self,
caller: &str,
pending: PendingPush,
delivered: bool,
) -> anyhow::Result<()> {
let mut st = self.lock();
if delivered {
for (task, turn) in &pending.turns {
if let Some(entry) = st.tasks.get_mut(task) {
let mut task = entry.task.clone();
task.delivered = task.delivered.max(*turn);
task.updated_ms = crate::store::now_ms();
self.store.save(&task)?;
entry.task = task;
}
}
}
let mut ledger = st.ledger.clone();
if delivered {
for (task, seq) in &pending.events {
if !st.tasks.contains_key(task) {
continue;
}
let cursor = ledger
.cursors
.entry(super::cursor_key(caller, task))
.or_default();
*cursor = (*cursor).max(*seq);
}
}
ledger.pending_pushes.remove(caller);
ledger.save(&self.opts.home)?;
st.ledger = ledger;
if delivered {
for rid in &pending.approvals {
if let Some(p) = st.approvals.get_mut(rid) {
p.delivered = true;
}
}
}
drop(st);
if delivered {
self.record_delivery(
caller,
"push",
&pending.marker,
&pending.shown,
&pending.turns,
);
}
self.log(&format!(
"{caller} push read-back: {}",
if delivered {
"delivered"
} else {
"absent; retrying"
}
));
Ok(())
}
pub(super) async fn reconcile_before_take(&self, caller: &str) -> anyhow::Result<()> {
let host = {
let st = self.lock();
st.ledger
.pending_pushes
.get(caller)
.and_then(|_| st.ledger.codex_hosts.get(caller))
.cloned()
};
if let Some(host) = host {
let mut remote = tokio::time::timeout(
PROBE,
Remote::connect(&host.exe, host.codex_home.as_deref()),
)
.await
.context("push read-back connection timed out; retry later")?
.context("push read-back connection failed; retry later")?;
self.reconcile_push(caller, &host.thread, &mut remote)
.await?;
}
Ok(())
}
async fn push(self: &Arc<Self>, caller: &str) {
let delivering = self.delivery(caller).lock_owned().await;
let needs_push = {
let st = self.lock();
!self.news(&st, caller, &[]).is_empty() || st.ledger.pending_pushes.contains_key(caller)
};
if !needs_push {
self.retract_doorbell(caller).await;
return;
}
let host = self.lock().ledger.codex_hosts.get(caller).cloned();
let Some(host) = host else {
return;
};
let outcome = self
.push_to_host(caller, &host, self.probe_host(caller, &host))
.await;
match outcome {
Ok((how, more)) => {
self.lock().push_failures.remove(caller);
self.log(&format!("push to {caller}: {how}"));
if more {
drop(delivering);
self.schedule_push(caller.to_string(), GATHER);
}
}
Err(e) => self.retry_push(caller, &e),
}
}
fn retry_push(self: &Arc<Self>, caller: &str, error: &str) {
let failures = {
let mut st = self.lock();
st.host_checked.remove(caller);
let n = st.push_failures.entry(caller.to_string()).or_default();
*n += 1;
*n
};
let delay = RETRY
.saturating_mul(1 << (failures - 1).min(5))
.min(RETRY_MAX);
self.log(&format!(
"{caller}: {error}; trying again in {}s",
delay.as_secs()
));
self.schedule_push(caller.to_string(), delay);
}
async fn push_to_host(
&self,
caller: &str,
host: &CodexHost,
probe: impl Future<Output = (Reach, Option<&'static str>)>,
) -> Result<(String, bool), String> {
let mut host = host.clone();
if self.cached_host(caller, &host).is_none() {
let (reach, why) = probe.await;
if reach == Reach::Wait {
return Err(why.unwrap_or("the host is unavailable").to_string());
}
host.doorbell = reach == Reach::Doorbell;
let mut st = self.lock();
let Some(saved) = st.ledger.codex_hosts.get_mut(caller) else {
return Ok(("the host was removed".to_string(), false));
};
let previous = std::mem::replace(saved, host.clone());
if let Err(e) = st.ledger.save(&self.opts.home) {
st.ledger.codex_hosts.insert(caller.to_string(), previous);
st.host_checked.remove(caller);
return Err(format!("saving the host route: {e:#}"));
}
Self::cache_host_check(&mut st, caller);
}
if host.doorbell {
self.reconcile_before_take(caller)
.await
.map_err(|e| format!("{e:#}"))?;
self.ring(caller, &host).await.map(|how| (how, false))
} else {
self.send_events(caller, &host).await
}
}
async fn send_events(&self, caller: &str, host: &CodexHost) -> Result<(String, bool), String> {
let mut remote = None;
let mut batch = None;
let sent = tokio::time::timeout(TIMEOUT, async {
let r = remote.insert(Remote::connect(&host.exe, host.codex_home.as_deref()).await?);
self.reconcile_push(caller, &host.thread, r).await?;
let (approvals, events, news) = {
let st = self.lock();
(
self.waiting_approvals(&st, caller, &[]),
self.unseen(&st, caller, &[]),
self.news(&st, caller, &[]),
)
};
if news.is_empty() {
return Ok(("no events requiring attention".to_string(), false));
}
if !r.is_open(&host.thread).await? {
anyhow::bail!("the session is closed");
}
let taken = self.batch(&approvals, &events);
if !self.worth_pushing(caller, &taken) {
return Ok(("nothing left to send".to_string(), false));
}
let texts: Vec<&str> = taken.items.iter().map(|(_, t)| t.as_str()).collect();
let marker = super::delivery::mark();
let message = format!(
"From your sub-tasks (sent by axiopera, not typed by the user):\n{}",
texts.join("\n")
);
let pending = PendingPush {
marker: marker.clone(),
events: taken.seen.iter().map(|e| (e.task.clone(), e.seq)).collect(),
shown: taken.shown(),
turns: taken.turns.clone(),
approvals: taken.approvals.clone(),
};
{
let mut st = self.lock();
let mut ledger = st.ledger.clone();
ledger.pending_pushes.insert(caller.to_string(), pending);
ledger.save(&self.opts.home)?;
st.ledger = ledger;
}
let more = taken.left > 0;
batch = Some((taken, marker.clone()));
let how = r.message(&host.thread, &message, &marker).await?;
anyhow::Ok((how.to_string(), more))
})
.await;
let outcome = match sent {
Ok(result) => result.map_err(|e| format!("{e:#}")),
Err(_) => Err("timed out".to_string()),
};
if outcome.is_ok() {
if let Some((b, marker)) = &batch {
if let Err(e) = self.mark_seen(caller, &b.seen, &b.turns, &b.approvals) {
return Err(format!("saving cursors: {e:#}"));
}
self.clear_pending_push(caller)
.map_err(|e| format!("saving push state: {e:#}"))?;
self.record_delivery(caller, "push", marker, &b.shown(), &b.turns);
}
} else if batch.is_some() && !remote.as_ref().is_some_and(Remote::sent) {
self.clear_pending_push(caller)
.map_err(|e| format!("saving push state: {e:#}"))?;
}
outcome
}
fn worth_pushing(&self, caller: &str, taken: &super::Batch) -> bool {
!taken.items.is_empty() && !self.news(&self.lock(), caller, &[]).is_empty()
}
async fn ring(&self, caller: &str, host: &CodexHost) -> Result<String, String> {
let rung = self.lock().ledger.doorbells.get(caller).cloned();
let result = tokio::time::timeout(TIMEOUT, async {
let mut r = Remote::private(&host.exe, host.codex_home.as_deref()).await?;
if let Some(id) = rung
&& r.queued(&host.thread, &id).await?
{
return Ok("its reminder is still queued".to_string());
}
let Some((news, _)) = self.reminder(caller) else {
return Ok("nothing to remind of".to_string());
};
let text =
format!("From your sub-tasks (sent by axiopera, not typed by the user):\n{news}");
let id = r
.enqueue(&host.thread, &text, &super::delivery::mark())
.await?;
let mut st = self.lock();
st.ledger.doorbells.insert(caller.to_string(), id);
st.ledger.save(&self.opts.home)?;
anyhow::Ok("queued a reminder".to_string())
})
.await;
match result {
Ok(result) => result.map_err(|e| format!("{e:#}")),
Err(_) => Err("timed out".to_string()),
}
}
pub(super) async fn retract_doorbell(self: &Arc<Self>, caller: &str) {
let (id, host) = {
let st = self.lock();
let Some(id) = st.ledger.doorbells.get(caller) else {
return;
};
if !self.news(&st, caller, &[]).is_empty() {
return;
}
let Some(host) = st.ledger.codex_hosts.get(caller) else {
return;
};
(id.clone(), host.clone())
};
let result = async {
let deleted = tokio::time::timeout(PROBE, async {
Remote::private(&host.exe, host.codex_home.as_deref())
.await?
.dequeue(&host.thread, &id)
.await
})
.await
.context("withdrawing reminder timed out")??;
let mut st = self.lock();
let mut ledger = st.ledger.clone();
ledger.doorbells.remove(caller);
ledger.save(&self.opts.home)?;
st.ledger = ledger;
st.push_failures.remove(caller);
anyhow::Ok(deleted)
}
.await;
match result {
Ok(true) => self.log(&format!("{caller} withdrew its queued reminder")),
Ok(false) => (),
Err(e) => self.retry_push(
caller,
&format!("could not withdraw queued reminder: {e:#}"),
),
}
}
}
#[cfg(test)]
mod tests {
use super::super::tests::fixture;
use super::*;
use crate::daemon::{Out, render};
use crate::store::Event;
#[test]
fn a_push_goes_out_while_news_follows_notes_and_not_once_gc_took_the_news() {
let f = fixture();
let note = "n".repeat(render::RESPONSE_TOKENS * 3);
f.d.emit(
"test",
"message",
serde_json::json!({"to": "sibling", "text": note}),
);
let result = "r".repeat(render::RESPONSE_TOKENS * 2);
f.d.store.save_turn("test", 1, &result).unwrap();
f.d.emit(
"test",
"turn_ended",
serde_json::json!({"turn": 1, "ok": true}),
);
let events = f.d.unseen(&f.d.lock(), "codex:host", &[]);
let taken = f.d.batch(&[], &events);
assert_eq!(taken.items.len(), 1, "the note fills the first push");
assert!(f.d.worth_pushing("codex:host", &taken));
f.d.collect(u64::MAX).unwrap();
assert!(!f.d.worth_pushing("codex:host", &taken));
}
#[test]
fn only_actionable_events_start_a_host_turn() {
let event = |kind: &str| Event {
seq: 1,
ts: 0,
task: "test".into(),
kind: kind.into(),
data: Default::default(),
};
for kind in [
"blocked",
"steered",
"input_read",
"tool",
"said",
"file_changed",
"message",
"detached",
] {
assert!(!is_news(&event(kind), "codex:host"));
}
for kind in [
"turn_ended",
"question",
"approval_request",
"stalled",
"failed",
"cancelled",
"interrupted",
] {
assert!(is_news(&event(kind), "codex:host"));
}
}
#[test]
fn cancellation_keeps_its_author_and_only_notifies_other_callers() {
for by in ["codex:host", "codex:other", "user", "timeout"] {
let f = fixture();
f.d.cancel_tree("test", "cancelled", by);
let events = f.d.unseen(&f.d.lock(), "codex:host", &[]);
assert_eq!(events.len(), 1);
assert_eq!(events[0].data["by"], by);
assert_eq!(is_news(&events[0], "codex:host"), by != "codex:host");
let saved = f.d.store.events("test");
assert_eq!(saved[0].data["by"], by);
assert_eq!(is_news(&saved[0], "codex:host"), by != "codex:host");
}
}
#[tokio::test]
async fn own_cancellation_stays_for_wait_or_a_later_push_without_connecting() {
let f = fixture();
f.d.lock().ledger.codex_hosts.insert(
"codex:host".into(),
CodexHost {
exe: f.d.opts.home.join("missing-codex"),
codex_home: Some(f.d.opts.home.clone()),
thread: "host".into(),
doorbell: false,
},
);
f.d.cancel_tree("test", "cancelled by request", "codex:host");
f.d.push("codex:host").await;
{
let st = f.d.lock();
assert!(st.push_failures.is_empty());
assert!(st.pushing.is_empty());
assert_eq!(f.d.unseen(&st, "codex:host", &[]).len(), 1);
}
f.d.emit("test", "stalled", serde_json::json!({"quiet_secs": 60}));
let events = f.d.unseen(&f.d.lock(), "codex:host", &[]);
assert_eq!(events.len(), 2);
assert!(events.iter().any(|e| is_news(e, "codex:host")));
assert_eq!(events[0].kind, "cancelled");
}
#[test]
fn settling_a_push_does_not_resurrect_collected_tasks_or_cursors() {
let f = fixture();
let pending = PendingPush {
marker: "receipt".into(),
events: vec![("gone".into(), 8), ("test".into(), 9)],
shown: vec![("gone".into(), 8), ("test".into(), 9)],
turns: vec![("gone".into(), 1), ("test".into(), 1)],
approvals: Vec::new(),
};
f.d.settle_push("codex:host", pending, true).unwrap();
let st = f.d.lock();
assert!(!st.tasks.contains_key("gone"));
assert!(!st.ledger.cursors.contains_key("codex:host|gone"));
assert_eq!(st.ledger.cursors["codex:host|test"], 9);
assert_eq!(st.tasks["test"].task.delivered, 1);
drop(st);
assert!(!f.d.store.exists("gone"));
}
#[test]
fn recovered_batch_advances_only_its_cursors_and_only_when_present() {
for found in [false, true] {
let f = fixture();
let pending = PendingPush {
marker: "axiopera:receipt".into(),
events: vec![("test".into(), 5), ("test".into(), 7)],
shown: vec![("test".into(), 7)],
turns: vec![("test".into(), 1)],
approvals: vec!["r1".into()],
};
let (reply, _decision) = tokio::sync::oneshot::channel();
{
let mut st = f.d.lock();
st.approvals.insert(
"r1".into(),
super::super::Pending {
task: "test".into(),
approver: "codex:host".into(),
summary: "Bash: ls".into(),
delivered: false,
reply,
},
);
st.ledger
.pending_pushes
.insert("codex:host".into(), pending);
st.ledger.save(&f.d.opts.home).unwrap();
}
let loaded = crate::store::Ledger::load(&f.d.opts.home).unwrap();
let pending = loaded.pending_pushes["codex:host"].clone();
assert_eq!(pending.marker, "axiopera:receipt");
f.d.settle_push("codex:host", pending, found).unwrap();
let st = f.d.lock();
assert!(st.ledger.pending_pushes.is_empty());
assert_eq!(st.tasks["test"].task.delivered, u32::from(found));
assert_eq!(
st.ledger.cursors.get("codex:host|test").copied(),
found.then_some(7)
);
assert_eq!(st.approvals["r1"].delivered, found);
drop(st);
let records: Vec<_> =
f.d.store
.events("test")
.into_iter()
.filter(|e| e.kind == "delivered")
.map(|e| serde_json::Value::Object(e.data))
.collect();
let expected = serde_json::json!({
"to": "codex:host", "via": "push", "batch": "axiopera:receipt",
"events": [7], "results": [1],
});
assert_eq!(records, if found { vec![expected] } else { vec![] });
}
}
#[test]
fn hosts_registered_before_doorbells_are_pushed_to() {
let host: CodexHost =
serde_json::from_value(serde_json::json!({"exe": "codex", "thread": "t"})).unwrap();
assert!(!host.doorbell);
assert!(
serde_json::to_value(&host)
.unwrap()
.get("doorbell")
.is_none()
);
}
#[tokio::test]
async fn failed_withdrawal_keeps_the_queue_id_for_retry_after_inbox_or_restart() {
let f = fixture();
{
let mut st = f.d.lock();
st.ledger.codex_hosts.insert(
"codex:host".into(),
CodexHost {
exe: f.d.opts.home.join("missing-codex"),
codex_home: Some(f.d.opts.home.clone()),
thread: "host".into(),
doorbell: true,
},
);
st.ledger
.doorbells
.insert("codex:host".into(), "queued".into());
}
f.d.emit("test", "stalled", serde_json::json!({"quiet_secs": 60}));
f.d.retract_doorbell("codex:host").await;
assert!(f.d.lock().pushing.is_empty());
f.d.inbox("codex:host", &[], &mut Out::new(&mut Vec::new()))
.await
.unwrap();
let saved = crate::store::Ledger::load(&f.d.opts.home).unwrap();
assert_eq!(saved.doorbells["codex:host"], "queued");
assert!(f.d.lock().pushing.contains_key("codex:host"));
let restored = Arc::new(
Daemon::load(super::super::options(f.d.opts.home.clone()).unwrap(), None).unwrap(),
);
restored.push("codex:host").await;
assert!(restored.lock().pushing.contains_key("codex:host"));
assert_eq!(restored.lock().ledger.doorbells["codex:host"], "queued");
}
#[tokio::test]
async fn withdrawal_is_attempted_once_only_handled_approvals_and_notes_remain() {
let f = fixture();
f.d.update("test", |t| t.tier = crate::ipc::Tier::Work);
let ring = || {
let mut st = f.d.lock();
st.ledger.codex_hosts.insert(
"codex:host".into(),
CodexHost {
exe: f.d.opts.home.join("missing-codex"),
codex_home: Some(f.d.opts.home.clone()),
thread: "host".into(),
doorbell: true,
},
);
st.ledger
.doorbells
.insert("codex:host".into(), "queued".into());
};
let request = |summary: &str| {
let (reply, decision) = tokio::sync::oneshot::channel();
f.d.request_approval(
"test",
crate::agent::Approval {
summary: summary.into(),
detail: format!(
"Bash
command: {summary}"
),
reply,
withdrawn: tokio::sync::oneshot::channel().1,
},
);
decision
};
let inbox = async || {
f.d.inbox("codex:host", &[], &mut Out::new(&mut Vec::new()))
.await
.unwrap()
};
ring();
let note = "x".repeat(render::RESPONSE_TOKENS * 4);
f.d.emit(
"test",
"message",
serde_json::json!({"to": "other", "text": note}),
);
let _first = request("rm -rf build");
inbox().await;
assert!(f.d.lock().pushing.contains_key("codex:host"));
inbox().await;
f.d.lock().pushing.clear();
ring();
let _second = request("ls");
let rid = f.d.lock().approvals.keys().next().unwrap().clone();
f.d.resolve(&rid, Some("user"), crate::agent::Decision::Approve)
.unwrap();
inbox().await;
assert!(f.d.lock().pushing.contains_key("codex:host"));
}
}
#[cfg(test)]
mod capability_cache_tests {
use super::*;
fn registered_host(f: &crate::daemon::tests::Fixture, doorbell: bool) -> CodexHost {
let host = CodexHost {
exe: f.d.opts.home.join("missing-codex"),
codex_home: Some(f.d.opts.home.clone()),
thread: "host".into(),
doorbell,
};
let mut st = f.d.lock();
st.ledger
.codex_hosts
.insert("codex:host".into(), host.clone());
st.ledger.save(&f.d.opts.home).unwrap();
Daemon::cache_host_check(&mut st, "codex:host");
host
}
#[tokio::test]
async fn inbox_keeps_an_unconfirmed_batch_when_read_back_is_unavailable() {
let f = crate::daemon::tests::fixture();
registered_host(&f, false);
f.d.store
.save_turn("test", 1, "undelivered result")
.unwrap();
f.d.try_update("test", |t| t.turns = 1).unwrap();
f.d.try_emit(
"test",
"turn_ended",
serde_json::json!({"turn": 1, "ok": true}),
)
.unwrap();
{
let mut st = f.d.lock();
let seq = st.seq;
st.ledger.pending_pushes.insert(
"codex:host".into(),
PendingPush {
marker: "saved-before-send".into(),
events: vec![("test".into(), seq)],
shown: vec![],
turns: vec![("test".into(), 1)],
approvals: vec![],
},
);
st.ledger.save(&f.d.opts.home).unwrap();
}
let before = std::fs::read(f.d.opts.home.join("ledger.json")).unwrap();
let restored = Arc::new(
Daemon::load(super::super::options(f.d.opts.home.clone()).unwrap(), None).unwrap(),
);
let mut output = Vec::new();
for _ in 0..2 {
let error = restored
.inbox("codex:host", &[], &mut super::super::Out::new(&mut output))
.await
.unwrap_err();
assert!(error.to_string().contains("retry later"));
assert!(output.is_empty());
let st = restored.lock();
assert_eq!(st.tasks["test"].task.delivered, 0);
assert!(st.ledger.cursors.is_empty());
assert_eq!(
st.ledger.pending_pushes["codex:host"].marker,
"saved-before-send"
);
assert_eq!(restored.unseen(&st, "codex:host", &[]).len(), 1);
}
assert_eq!(
std::fs::read(f.d.opts.home.join("ledger.json")).unwrap(),
before
);
let pending = restored.lock().ledger.pending_pushes["codex:host"].clone();
restored.settle_push("codex:host", pending, false).unwrap();
restored
.inbox("codex:host", &[], &mut super::super::Out::new(&mut output))
.await
.unwrap();
assert!(
String::from_utf8(output)
.unwrap()
.contains("undelivered result")
);
assert_eq!(restored.lock().tasks["test"].task.delivered, 1);
}
#[tokio::test(start_paused = true)]
async fn invalidated_pushes_probe_again_and_persist_either_route() {
for doorbell in [false, true] {
let f = crate::daemon::tests::fixture();
let host = registered_host(&f, doorbell);
f.d.emit("test", "stalled", serde_json::json!({"quiet_secs": 60}));
f.d.push("codex:host").await;
assert!(!f.d.lock().host_checked.contains_key("codex:host"));
assert_eq!(f.d.lock().push_failures["codex:host"], 1);
let mut probed = false;
let outcome =
f.d.push_to_host("codex:host", &host, async {
probed = true;
(
if doorbell {
Reach::Push
} else {
Reach::Doorbell
},
None,
)
})
.await;
assert!(probed);
assert!(outcome.is_err());
let ledger = crate::store::Ledger::load(&f.d.opts.home).unwrap();
assert_eq!(ledger.codex_hosts["codex:host"].doorbell, !doorbell);
assert!(ledger.cursors.is_empty());
assert_eq!(f.d.unseen(&f.d.lock(), "codex:host", &[]).len(), 1);
}
}
#[tokio::test]
async fn a_valid_cached_route_does_not_probe_again() {
let f = crate::daemon::tests::fixture();
let host = registered_host(&f, false);
let outcome =
f.d.push_to_host("codex:host", &host, async {
panic!("a valid cache must skip the probe");
})
.await;
assert!(outcome.is_err());
}
#[tokio::test(start_paused = true)]
async fn unavailable_reprobes_preserve_the_route_and_exponential_retry() {
let f = crate::daemon::tests::fixture();
registered_host(&f, false);
f.d.emit("test", "stalled", serde_json::json!({"quiet_secs": 60}));
f.d.push("codex:host").await;
assert_eq!(f.d.lock().pushing["codex:host"], Instant::now() + RETRY);
tokio::task::yield_now().await;
tokio::time::advance(RETRY).await;
for _ in 0..10 {
tokio::task::yield_now().await;
}
let st = f.d.lock();
assert_eq!(st.push_failures["codex:host"], 2);
assert_eq!(st.pushing["codex:host"], Instant::now() + RETRY * 2);
assert!(!st.host_checked.contains_key("codex:host"));
assert!(!st.ledger.codex_hosts["codex:host"].doorbell);
assert!(st.ledger.cursors.is_empty());
}
#[tokio::test]
async fn switching_to_doorbell_reconciles_the_written_batch_before_ringing() {
let f = crate::daemon::tests::fixture();
let host = registered_host(&f, false);
f.d.emit("test", "stalled", serde_json::json!({"quiet_secs": 60}));
{
let mut st = f.d.lock();
st.host_checked.clear();
st.ledger.pending_pushes.insert(
"codex:host".into(),
PendingPush {
marker: "already-written".into(),
events: vec![("test".into(), 1)],
shown: vec![],
turns: vec![("test".into(), 1)],
approvals: vec![],
},
);
st.ledger
.doorbells
.insert("codex:host".into(), "existing-reminder".into());
st.ledger.save(&f.d.opts.home).unwrap();
}
let outcome =
f.d.push_to_host("codex:host", &host, async { (Reach::Doorbell, None) })
.await;
assert!(outcome.is_err());
let ledger = crate::store::Ledger::load(&f.d.opts.home).unwrap();
assert!(ledger.codex_hosts["codex:host"].doorbell);
assert!(ledger.pending_pushes.contains_key("codex:host"));
assert!(ledger.cursors.is_empty());
assert_eq!(ledger.doorbells["codex:host"], "existing-reminder");
assert_eq!(f.d.lock().tasks["test"].task.delivered, 0);
}
#[tokio::test]
async fn a_failed_route_save_preserves_the_old_route_and_pending_batch() {
let f = crate::daemon::tests::fixture();
let host = registered_host(&f, false);
{
let mut st = f.d.lock();
st.host_checked.clear();
st.ledger.pending_pushes.insert(
"codex:host".into(),
PendingPush {
marker: "keep-for-read-back".into(),
events: vec![("test".into(), 1)],
shown: vec![],
turns: vec![],
approvals: vec![],
},
);
st.ledger.save(&f.d.opts.home).unwrap();
}
std::fs::create_dir(f.d.opts.home.join("ledger.json.tmp")).unwrap();
let error =
f.d.push_to_host("codex:host", &host, async { (Reach::Doorbell, None) })
.await
.unwrap_err();
assert!(error.starts_with("saving the host route:"));
let st = f.d.lock();
assert!(!st.ledger.codex_hosts["codex:host"].doorbell);
assert!(!st.host_checked.contains_key("codex:host"));
assert_eq!(
st.ledger.pending_pushes["codex:host"].marker,
"keep-for-read-back"
);
let disk = crate::store::Ledger::load(&f.d.opts.home).unwrap();
assert_eq!(
serde_json::to_value(&st.ledger).unwrap(),
serde_json::to_value(disk).unwrap()
);
}
#[tokio::test]
async fn host_registration_waits_for_an_in_progress_delivery() {
let f = crate::daemon::tests::fixture();
let held = f.d.delivery("codex:host").lock_owned().await;
let d = f.d.clone();
let registering = tokio::spawn(async move { d.register_host("codex:host", &[]).await });
tokio::task::yield_now().await;
assert!(!registering.is_finished());
drop(held);
assert_eq!(registering.await.unwrap().0, Reach::Wait);
}
#[tokio::test]
async fn a_reprobe_does_not_restore_a_host_removed_while_awaiting_it() {
let f = crate::daemon::tests::fixture();
let host = registered_host(&f, false);
f.d.lock().host_checked.clear();
let outcome =
f.d.push_to_host("codex:host", &host, async {
f.d.lock().ledger.codex_hosts.remove("codex:host");
(Reach::Doorbell, None)
})
.await;
assert_eq!(
outcome.unwrap(),
("the host was removed".to_string(), false)
);
assert!(!f.d.lock().ledger.codex_hosts.contains_key("codex:host"));
assert!(!f.d.lock().host_checked.contains_key("codex:host"));
}
#[test]
fn only_recent_successes_with_matching_launch_identity_are_reused() {
let f = crate::daemon::tests::fixture();
let mut host = CodexHost {
exe: "fake-codex".into(),
codex_home: None,
thread: "host".into(),
doorbell: false,
};
f.d.lock()
.ledger
.codex_hosts
.insert("codex:host".into(), host.clone());
assert_eq!(f.d.cached_host("codex:host", &host), None);
f.d.lock()
.host_checked
.insert("codex:host".into(), std::time::Instant::now());
assert_eq!(f.d.cached_host("codex:host", &host), Some(Reach::Push));
f.d.lock()
.ledger
.codex_hosts
.get_mut("codex:host")
.unwrap()
.doorbell = true;
assert_eq!(f.d.cached_host("codex:host", &host), Some(Reach::Doorbell));
host.codex_home = Some("different-home".into());
assert_eq!(f.d.cached_host("codex:host", &host), None);
host.codex_home = None;
host.exe = "changed-executable".into();
assert_eq!(f.d.cached_host("codex:host", &host), None);
host.exe = "fake-codex".into();
f.d.lock()
.host_checked
.insert("codex:host".into(), std::time::Instant::now() - HOST_TTL);
assert_eq!(f.d.cached_host("codex:host", &host), None);
}
}