use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use tokio::time::Instant;
use serde_json::Value;
use tokio::sync::broadcast;
use super::{Daemon, RELEASED, is_news};
use crate::agent::{self, codex_remote::Remote};
use crate::ipc::Agent;
use crate::store::{CodexHost, PendingPush};
const GATHER: Duration = Duration::from_secs(1);
const PROBE: Duration = Duration::from_secs(5);
const TIMEOUT: Duration = Duration::from_secs(20);
const RETRY: Duration = Duration::from_secs(30);
const RETRY_MAX: Duration = Duration::from_secs(600);
#[derive(Clone, Copy, PartialEq, Debug, serde::Serialize)]
#[serde(rename_all = "lowercase")]
pub(super) enum Reach {
Push,
Doorbell,
Wait,
}
impl Daemon {
pub(super) async fn register_host(
&self,
caller: &str,
env: &[(String, String)],
) -> (Reach, Option<&'static str>) {
let Some(thread) = caller.strip_prefix("codex:") else {
return (Reach::Wait, None);
};
let configured = self.opts.agents.get(Agent::Codex).path.as_deref();
let Ok(exe) = agent::resolve_exe(Agent::Codex, configured, env) else {
return (Reach::Wait, Some("Codex was not found"));
};
let codex_home = env
.iter()
.find(|(k, _)| k == "CODEX_HOME")
.map(|(_, v)| PathBuf::from(v));
let mut host = CodexHost {
exe,
codex_home,
thread: thread.to_string(),
doorbell: false,
};
let home = host.codex_home.as_deref();
let shared = self
.probe(caller, "is not on the shared Codex daemon", async {
Remote::connect(&host.exe, home)
.await?
.is_open(thread)
.await
})
.await;
let reach = if shared == Some(true) {
Reach::Push
} else {
let queued = self
.probe(caller, "has no Codex queue", async {
Remote::private(&host.exe, home)
.await?
.serves_queue(thread)
.await
})
.await;
let why = match (shared, queued) {
(_, Some(true)) => None,
(Some(false), Some(false)) => Some(
"it is neither on the shared Codex daemon nor open in the Codex app or IDE extension",
),
_ => Some("Codex did not answer within 5 seconds"),
};
if why.is_some() {
return (Reach::Wait, why);
}
Reach::Doorbell
};
host.doorbell = reach == Reach::Doorbell;
let mut st = self.lock();
st.ledger.codex_hosts.insert(caller.to_string(), host);
if let Err(e) = st.ledger.save(&self.opts.home) {
self.log(&format!("saving the ledger: {e:#}"));
}
(reach, None)
}
async fn probe(
&self,
caller: &str,
failure: &str,
check: impl Future<Output = anyhow::Result<bool>>,
) -> Option<bool> {
match tokio::time::timeout(PROBE, check).await {
Ok(Ok(yes)) => Some(yes),
Ok(Err(e)) => {
self.log(&format!("{caller} {failure}: {e:#}"));
Some(false)
}
Err(_) => {
self.log(&format!("{caller} {failure}: Codex did not answer"));
None
}
}
}
pub(super) async fn pusher(self: Arc<Self>) {
let mut rx = self.events.subscribe();
self.push_all();
loop {
let e = match rx.recv().await {
Ok(e) => e,
Err(broadcast::error::RecvError::Lagged(_)) => {
self.push_all();
continue;
}
Err(broadcast::error::RecvError::Closed) => return,
};
if e.kind == RELEASED {
continue;
}
let target = {
let st = self.lock();
if e.kind == "approval_request" {
e.data
.get("approver")
.and_then(Value::as_str)
.map(str::to_string)
} else {
st.tasks.get(&e.task).map(|t| t.task.parent.clone())
}
};
if let Some(target) = target
&& is_news(&e, &target)
{
self.schedule_push(target, GATHER);
}
}
}
fn push_all(self: &Arc<Self>) {
let hosts: Vec<String> = self.lock().ledger.codex_hosts.keys().cloned().collect();
for host in hosts {
self.schedule_push(host, GATHER);
}
}
fn schedule_push(self: &Arc<Self>, caller: String, delay: Duration) {
let due = Instant::now() + delay;
{
let mut st = self.lock();
if !st.ledger.codex_hosts.contains_key(&caller)
|| st.pushing.get(&caller).is_some_and(|d| *d <= due)
{
return;
}
st.pushing.insert(caller.clone(), due);
}
let d = self.clone();
tokio::spawn(async move {
tokio::time::sleep_until(due).await;
{
let mut st = d.lock();
if st.pushing.get(&caller) != Some(&due) {
return;
}
st.pushing.remove(&caller);
}
d.push(&caller).await;
});
}
fn clear_pending_push(&self, caller: &str) {
let mut st = self.lock();
st.ledger.pending_pushes.remove(caller);
if let Err(e) = st.ledger.save(&self.opts.home) {
self.log(&format!("saving push state: {e:#}"));
}
}
async fn reconcile_push(
&self,
caller: &str,
thread: &str,
remote: &mut Remote,
) -> anyhow::Result<()> {
let pending = self.lock().ledger.pending_pushes.get(caller).cloned();
let Some(pending) = pending else {
return Ok(());
};
let found =
tokio::time::timeout(PROBE, remote.contains_message(thread, &pending.marker)).await;
let delivered = match found {
Ok(Ok(found)) => found,
other => {
self.log(&format!("{caller} push read-back failed ({other:?}); treating the written batch as delivered"));
true
}
};
self.settle_push(caller, pending, delivered)
}
fn settle_push(
&self,
caller: &str,
pending: PendingPush,
delivered: bool,
) -> anyhow::Result<()> {
if delivered {
for (task, turn) in pending.turns {
self.update(&task, |t| t.delivered = t.delivered.max(turn));
}
}
let mut st = self.lock();
if delivered {
for (task, seq) in pending.events {
let cursor = st
.ledger
.cursors
.entry(super::cursor_key(caller, &task))
.or_default();
*cursor = (*cursor).max(seq);
}
for rid in &pending.approvals {
if let Some(p) = st.approvals.get_mut(rid) {
p.delivered = true;
}
}
}
st.ledger.pending_pushes.remove(caller);
st.ledger.save(&self.opts.home)?;
self.log(&format!(
"{caller} push read-back: {}",
if delivered {
"delivered"
} else {
"absent; retrying"
}
));
Ok(())
}
pub(super) async fn reconcile_before_take(&self, caller: &str) -> anyhow::Result<()> {
let pair = {
let st = self.lock();
st.ledger
.pending_pushes
.get(caller)
.cloned()
.zip(st.ledger.codex_hosts.get(caller).cloned())
};
if let Some((pending, host)) = pair {
match tokio::time::timeout(
PROBE,
Remote::connect(&host.exe, host.codex_home.as_deref()),
)
.await
{
Ok(Ok(mut remote)) => {
self.reconcile_push(caller, &host.thread, &mut remote)
.await?
}
_ => {
self.log(&format!("{caller} push read-back unavailable; treating the written batch as delivered"));
self.settle_push(caller, pending, true)?;
}
}
}
Ok(())
}
async fn push(self: &Arc<Self>, caller: &str) {
let delivering = self.delivery(caller).lock_owned().await;
{
let st = self.lock();
if !self
.unseen(&st, caller, &[])
.iter()
.any(|e| is_news(e, caller))
&& !st.ledger.pending_pushes.contains_key(caller)
{
return;
}
}
let host = self.lock().ledger.codex_hosts.get(caller).cloned();
let Some(host) = host else {
return;
};
let outcome = if host.doorbell {
self.ring(caller, &host).await.map(|how| (how, false))
} else {
self.send_events(caller, &host).await
};
match outcome {
Ok((how, more)) => {
self.lock().push_failures.remove(caller);
self.log(&format!("push to {caller}: {how}"));
if more {
drop(delivering);
self.schedule_push(caller.to_string(), GATHER);
}
}
Err(e) => {
let failures = {
let mut st = self.lock();
let n = st.push_failures.entry(caller.to_string()).or_default();
*n += 1;
*n
};
let delay = RETRY
.saturating_mul(1 << (failures - 1).min(5))
.min(RETRY_MAX);
self.log(&format!(
"{caller} kept its events: {e}; trying again in {}s",
delay.as_secs()
));
drop(delivering);
self.schedule_push(caller.to_string(), delay);
}
}
}
async fn send_events(&self, caller: &str, host: &CodexHost) -> Result<(String, bool), String> {
let mut remote = None;
let mut batch = None;
let sent = tokio::time::timeout(TIMEOUT, async {
let r = remote.insert(Remote::connect(&host.exe, host.codex_home.as_deref()).await?);
self.reconcile_push(caller, &host.thread, r).await?;
let (approvals, events, news) = {
let st = self.lock();
(
self.waiting_approvals(&st, caller, &[]),
self.unseen(&st, caller, &[]),
self.news(&st, caller, &[]),
)
};
if news.is_empty() {
return Ok(("no events requiring attention".to_string(), false));
}
if !r.is_open(&host.thread).await? {
anyhow::bail!("the session is closed");
}
let taken = self.batch(&approvals, &events);
if !self.worth_pushing(caller, &taken) {
return Ok(("nothing left to send".to_string(), false));
}
let texts: Vec<&str> = taken.items.iter().map(|(_, t)| t.as_str()).collect();
let marker = format!(
"[axiopera:{}]",
&uuid::Uuid::new_v4().simple().to_string()[..12]
);
let message = format!(
"{marker} From your sub-tasks (sent by axiopera, not typed by the user):\n{}",
texts.join("\n")
);
let pending = PendingPush {
marker,
events: taken.seen.iter().map(|e| (e.task.clone(), e.seq)).collect(),
turns: taken.turns.clone(),
approvals: taken.approvals.clone(),
};
{
let mut st = self.lock();
st.ledger.pending_pushes.insert(caller.to_string(), pending);
st.ledger.save(&self.opts.home)?;
}
let more = taken.left > 0;
batch = Some(taken);
let how = r.message(&host.thread, &message).await?;
anyhow::Ok((how.to_string(), more))
})
.await;
let outcome = match sent {
Ok(result) => result.map_err(|e| format!("{e:#}")),
Err(_) => Err("timed out".to_string()),
};
if outcome.is_ok() {
if let Some(b) = &batch {
if let Err(e) = self.mark_seen(caller, &b.seen, &b.turns, &b.approvals) {
self.log(&format!("saving cursors: {e:#}"));
}
self.clear_pending_push(caller);
}
} else if batch.is_some() && !remote.as_ref().is_some_and(Remote::sent) {
self.clear_pending_push(caller);
}
outcome
}
fn worth_pushing(&self, caller: &str, taken: &super::Batch) -> bool {
!taken.items.is_empty() && !self.news(&self.lock(), caller, &[]).is_empty()
}
async fn ring(&self, caller: &str, host: &CodexHost) -> Result<String, String> {
let rung = self.lock().ledger.doorbells.get(caller).cloned();
let result = tokio::time::timeout(TIMEOUT, async {
let mut r = Remote::private(&host.exe, host.codex_home.as_deref()).await?;
if let Some(id) = rung
&& r.queued(&host.thread, &id).await?
{
return Ok("its reminder is still queued".to_string());
}
let Some((news, _)) = self.reminder(caller) else {
return Ok("nothing to remind of".to_string());
};
let text =
format!("From your sub-tasks (sent by axiopera, not typed by the user):\n{news}");
let id = r.enqueue(&host.thread, &text).await?;
let mut st = self.lock();
st.ledger.doorbells.insert(caller.to_string(), id);
st.ledger.save(&self.opts.home)?;
anyhow::Ok("queued a reminder".to_string())
})
.await;
match result {
Ok(result) => result.map_err(|e| format!("{e:#}")),
Err(_) => Err("timed out".to_string()),
}
}
pub(super) fn retract_doorbell(self: &Arc<Self>, caller: &str) {
let (id, host) = {
let mut st = self.lock();
if !self.news(&st, caller, &[]).is_empty() {
return;
}
let Some(id) = st.ledger.doorbells.remove(caller) else {
return;
};
if let Err(e) = st.ledger.save(&self.opts.home) {
self.log(&format!("saving the ledger: {e:#}"));
}
(id, st.ledger.codex_hosts.get(caller).cloned())
};
let Some(host) = host else {
return;
};
let d = self.clone();
let caller = caller.to_string();
tokio::spawn(async move {
let deleted = tokio::time::timeout(TIMEOUT, async {
Remote::private(&host.exe, host.codex_home.as_deref())
.await?
.dequeue(&host.thread, &id)
.await
})
.await;
let how = match deleted {
Ok(Ok(true)) => "withdrew its queued reminder".to_string(),
Ok(Ok(false)) => return,
Ok(Err(e)) => format!("could not withdraw its queued reminder: {e:#}"),
Err(_) => "could not withdraw its queued reminder: timed out".to_string(),
};
d.log(&format!("{caller} {how}"));
});
}
}
#[cfg(test)]
mod tests {
use super::super::tests::fixture;
use super::*;
use crate::daemon::{Out, render};
use crate::store::Event;
#[test]
fn a_push_goes_out_while_news_follows_notes_and_not_once_gc_took_the_news() {
let f = fixture();
let note = "n".repeat(render::RESPONSE_TOKENS * 3);
f.d.emit(
"test",
"message",
serde_json::json!({"to": "sibling", "text": note}),
);
let result = "r".repeat(render::RESPONSE_TOKENS * 2);
f.d.store.save_turn("test", 1, &result).unwrap();
f.d.emit(
"test",
"turn_ended",
serde_json::json!({"turn": 1, "ok": true}),
);
let events = f.d.unseen(&f.d.lock(), "codex:host", &[]);
let taken = f.d.batch(&[], &events);
assert_eq!(taken.items.len(), 1, "the note fills the first push");
assert!(f.d.worth_pushing("codex:host", &taken));
f.d.collect(u64::MAX).unwrap();
assert!(!f.d.worth_pushing("codex:host", &taken));
}
#[test]
fn only_actionable_events_start_a_host_turn() {
let event = |kind: &str| Event {
seq: 1,
ts: 0,
task: "test".into(),
kind: kind.into(),
data: Default::default(),
};
for kind in [
"blocked",
"steered",
"input_read",
"tool",
"file_changed",
"message",
"detached",
] {
assert!(!is_news(&event(kind), "codex:host"));
}
for kind in [
"turn_ended",
"question",
"approval_request",
"stalled",
"failed",
"cancelled",
"interrupted",
] {
assert!(is_news(&event(kind), "codex:host"));
}
}
#[test]
fn cancellation_keeps_its_author_and_only_notifies_other_callers() {
for by in ["codex:host", "codex:other", "user", "timeout"] {
let f = fixture();
f.d.cancel_tree("test", "cancelled", by);
let events = f.d.unseen(&f.d.lock(), "codex:host", &[]);
assert_eq!(events.len(), 1);
assert_eq!(events[0].data["by"], by);
assert_eq!(is_news(&events[0], "codex:host"), by != "codex:host");
let saved = f.d.store.events("test");
assert_eq!(saved[0].data["by"], by);
assert_eq!(is_news(&saved[0], "codex:host"), by != "codex:host");
}
}
#[tokio::test]
async fn own_cancellation_stays_for_wait_or_a_later_push_without_connecting() {
let f = fixture();
f.d.lock().ledger.codex_hosts.insert(
"codex:host".into(),
CodexHost {
exe: f.d.opts.home.join("missing-codex"),
codex_home: Some(f.d.opts.home.clone()),
thread: "host".into(),
doorbell: false,
},
);
f.d.cancel_tree("test", "cancelled by request", "codex:host");
f.d.push("codex:host").await;
{
let st = f.d.lock();
assert!(st.push_failures.is_empty());
assert!(st.pushing.is_empty());
assert_eq!(f.d.unseen(&st, "codex:host", &[]).len(), 1);
}
f.d.emit("test", "stalled", serde_json::json!({"quiet_secs": 60}));
let events = f.d.unseen(&f.d.lock(), "codex:host", &[]);
assert_eq!(events.len(), 2);
assert!(events.iter().any(|e| is_news(e, "codex:host")));
assert_eq!(events[0].kind, "cancelled");
}
#[test]
fn recovered_batch_advances_only_its_cursors_and_only_when_present() {
for found in [false, true] {
let f = fixture();
let pending = PendingPush {
marker: "[axiopera:receipt]".into(),
events: vec![("test".into(), 7)],
turns: vec![("test".into(), 1)],
approvals: vec!["r1".into()],
};
let (reply, _decision) = tokio::sync::oneshot::channel();
{
let mut st = f.d.lock();
st.approvals.insert(
"r1".into(),
super::super::Pending {
task: "test".into(),
approver: "codex:host".into(),
summary: "Bash: ls".into(),
delivered: false,
reply,
},
);
st.ledger
.pending_pushes
.insert("codex:host".into(), pending);
st.ledger.save(&f.d.opts.home).unwrap();
}
let loaded = crate::store::Ledger::load(&f.d.opts.home);
let pending = loaded.pending_pushes["codex:host"].clone();
assert_eq!(pending.marker, "[axiopera:receipt]");
f.d.settle_push("codex:host", pending, found).unwrap();
let st = f.d.lock();
assert!(st.ledger.pending_pushes.is_empty());
assert_eq!(st.tasks["test"].task.delivered, u32::from(found));
assert_eq!(
st.ledger.cursors.get("codex:host|test").copied(),
found.then_some(7)
);
assert_eq!(st.approvals["r1"].delivered, found);
}
}
#[test]
fn hosts_registered_before_doorbells_are_pushed_to() {
let host: CodexHost =
serde_json::from_value(serde_json::json!({"exe": "codex", "thread": "t"})).unwrap();
assert!(!host.doorbell);
assert!(
serde_json::to_value(&host)
.unwrap()
.get("doorbell")
.is_none()
);
}
#[tokio::test]
async fn waiting_withdraws_the_reminder_once_nothing_needs_the_host() {
let f = fixture();
{
let mut st = f.d.lock();
st.ledger.codex_hosts.insert(
"codex:host".into(),
CodexHost {
exe: f.d.opts.home.join("missing-codex"),
codex_home: Some(f.d.opts.home.clone()),
thread: "host".into(),
doorbell: true,
},
);
st.ledger
.doorbells
.insert("codex:host".into(), "queued".into());
}
f.d.emit("test", "stalled", serde_json::json!({"quiet_secs": 60}));
f.d.retract_doorbell("codex:host");
assert!(f.d.lock().ledger.doorbells.contains_key("codex:host"));
let events = f.d.unseen(&f.d.lock(), "codex:host", &[]);
f.d.mark_seen("codex:host", &events, &[], &[]).unwrap();
f.d.retract_doorbell("codex:host");
assert!(f.d.lock().ledger.doorbells.is_empty());
assert!(
crate::store::Ledger::load(&f.d.opts.home)
.doorbells
.is_empty()
);
}
#[tokio::test]
async fn the_reminder_goes_once_only_requests_handed_over_or_decided_are_left() {
let f = fixture();
f.d.update("test", |t| t.tier = crate::ipc::Tier::Work);
let ring = || {
let mut st = f.d.lock();
st.ledger.codex_hosts.insert(
"codex:host".into(),
CodexHost {
exe: f.d.opts.home.join("missing-codex"),
codex_home: Some(f.d.opts.home.clone()),
thread: "host".into(),
doorbell: true,
},
);
st.ledger
.doorbells
.insert("codex:host".into(), "queued".into());
};
let request = |summary: &str| {
let (reply, decision) = tokio::sync::oneshot::channel();
f.d.request_approval(
"test",
crate::agent::Approval {
summary: summary.into(),
detail: format!(
"Bash
command: {summary}"
),
reply,
},
);
decision
};
let inbox = async || {
f.d.inbox("codex:host", &[], &mut Out::new(&mut Vec::new()))
.await
.unwrap()
};
ring();
let note = "x".repeat(render::RESPONSE_TOKENS * 4);
f.d.emit(
"test",
"message",
serde_json::json!({"to": "other", "text": note}),
);
let _first = request("rm -rf build");
inbox().await;
assert!(f.d.lock().ledger.doorbells.is_empty());
inbox().await;
ring();
let _second = request("ls");
let rid = f.d.lock().approvals.keys().next().unwrap().clone();
f.d.resolve(&rid, Some("user"), crate::agent::Decision::Approve)
.unwrap();
inbox().await;
assert!(f.d.lock().ledger.doorbells.is_empty());
}
}