use std::{sync::Arc, sync::OnceLock, time::Duration};
use serde_json::{Value, json};
use time::format_description::well_known::Rfc3339;
use tokio::{runtime::Handle, sync::mpsc::UnboundedReceiver, task::JoinSet};
use crate::{
configuration::{BundleMember, SessionType},
envelope::PromptBatchSettings,
runtime::{inscriptions::emit_inscription, signals::shutdown_requested},
};
use super::super::super::canonical_session_id;
use super::super::super::startup_state::note_session_served_successfully;
use super::super::super::stream::{
RelayStreamEvent, StreamEventSendOutcome, broadcast_event_to_bundle_ui,
list_registered_ui_sessions_for_bundle, send_event_to_registered_ui,
};
use super::super::super::{
AsyncDeliveryTask, DeliveryPayloadMode, RelayError, SendOutcome, SendResult,
};
use super::super::async_worker::{
AsyncWorkerKey, install_acp_worker_output_view, set_worker_readiness,
};
use super::super::choice_state::{
ChoiceEventContext, build_acp_chooser, invalidate_pending_for_respawn,
};
use super::super::quiescence::QUIESCENCE_TIMEOUT_MS_DEFAULT;
use super::payload::{
build_delivery_message, emit_envelope_metadata_inscription, resolve_target_member,
target_is_relay_wide,
};
use crate::transports::{
AcpDriverServices, ChoiceMade, ChoiceToMake, Chooser, DeliveryEnvelope, DeliveryMessage,
OutcomeFuture, SingleDeliveryOutcome, StartupContext, TransportImpl, UiBroadcastStatus,
UiIncomingMessage, UiOutcomePhase, UiTransportServices,
};
const ASYNC_WORKER_POLL_INTERVAL_MS: u64 = 100;
type InflightOutcome = (AsyncDeliveryTask, bool, Option<SingleDeliveryOutcome>);
#[derive(Clone)]
pub(super) struct AcpWorkerBootstrap {
pub(super) target_member: BundleMember,
pub(super) runtime_directory: std::path::PathBuf,
pub(super) choices_pending_max: usize,
}
pub(super) fn spawn_async_delivery_worker(
key: AsyncWorkerKey,
receiver: UnboundedReceiver<AsyncDeliveryTask>,
pending: std::sync::Arc<std::sync::atomic::AtomicUsize>,
bootstrap: Option<AcpWorkerBootstrap>,
) {
delivery_runtime_handle().spawn(async move {
run_async_delivery_worker(key, receiver, pending, bootstrap).await;
});
}
fn delivery_runtime_handle() -> Handle {
if let Ok(handle) = Handle::try_current() {
return handle;
}
static DELIVERY_RUNTIME: OnceLock<tokio::runtime::Runtime> = OnceLock::new();
DELIVERY_RUNTIME
.get_or_init(|| {
tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.enable_all()
.thread_name("agentmux-delivery")
.build()
.expect("build agentmux delivery fallback runtime")
})
.handle()
.clone()
}
async fn run_async_delivery_worker(
key: AsyncWorkerKey,
mut receiver: UnboundedReceiver<AsyncDeliveryTask>,
pending: std::sync::Arc<std::sync::atomic::AtomicUsize>,
bootstrap: Option<AcpWorkerBootstrap>,
) {
let batch_settings = super::prompt_batch_settings();
let mut transport: Option<TransportImpl> = match bootstrap {
Some(bootstrap) => {
let services = build_acp_driver_services(&key, &bootstrap);
let mut transport = TransportImpl::acp(
bootstrap.target_member,
bootstrap.runtime_directory,
key.namespace.clone(),
services,
batch_settings,
);
if let TransportImpl::Acp(driver) = &mut transport {
driver.bootstrap().await;
}
Some(transport)
}
None => None,
};
let poll_interval = Duration::from_millis(ASYNC_WORKER_POLL_INTERVAL_MS);
let mut inflight: JoinSet<InflightOutcome> = JoinSet::new();
let mut senders_dropped = false;
loop {
if shutdown_requested() {
shutdown_drain(
transport.as_mut(),
&mut inflight,
&mut receiver,
pending.as_ref(),
)
.await;
break;
}
if senders_dropped && inflight.is_empty() {
break;
}
tokio::select! {
maybe_task = receiver.recv(), if !senders_dropped => {
match maybe_task {
Some(task) => {
if shutdown_requested() {
super::super::async_worker::complete_task_on_shutdown(&task);
super::super::async_worker::release_pending_slot(pending.as_ref());
continue;
}
submit_task(
task,
&key,
&mut transport,
batch_settings,
&mut inflight,
pending.as_ref(),
);
}
None => senders_dropped = true,
}
}
joined = inflight.join_next(), if !inflight.is_empty() => {
if let Some(joined) = joined {
collect_outcome(joined, pending.as_ref());
}
}
_ = tokio::time::sleep(poll_interval) => {
}
}
}
super::super::async_worker::unregister_worker(&key);
}
fn submit_task(
task: AsyncDeliveryTask,
key: &AsyncWorkerKey,
transport: &mut Option<TransportImpl>,
batch_settings: PromptBatchSettings,
inflight: &mut JoinSet<InflightOutcome>,
pending: &std::sync::atomic::AtomicUsize,
) {
if transport.is_none() {
match build_worker_transport(&task, key, batch_settings) {
Ok(built) => *transport = Some(built),
Err(error) => {
super::super::async_worker::complete_task_outcome(&task, Err(error));
super::super::async_worker::release_pending_slot(pending);
return;
}
}
}
let transport = transport
.as_mut()
.expect("worker transport constructed above");
let (future, record_served) = if matches!(transport, TransportImpl::Pubsub) {
super::super::async_worker::complete_task_outcome(
&task,
Err(super::super::super::session_type_not_implemented(
task.target_session.as_str(),
SessionType::Pubsub,
)),
);
super::super::async_worker::release_pending_slot(pending);
return;
} else if matches!(transport, TransportImpl::Ui(_)) {
(transport.mailw(build_ui_envelope(&task)), false)
} else {
match prepare_coder_write(&task, transport) {
Ok(future) => (future, true),
Err(error) => {
super::super::async_worker::complete_task_outcome(&task, Err(error));
super::super::async_worker::release_pending_slot(pending);
return;
}
}
};
inflight.spawn(async move { (task, record_served, future.await.ok()) });
}
fn build_worker_transport(
task: &AsyncDeliveryTask,
key: &AsyncWorkerKey,
batch_settings: PromptBatchSettings,
) -> Result<TransportImpl, RelayError> {
if target_is_relay_wide(task) {
return Ok(TransportImpl::ui(build_ui_transport_services(key)));
}
let target_member =
resolve_target_member(task)?.expect("configured non-relay-wide target must have a member");
match target_member.target.session_type() {
SessionType::Tmux => {
let mut transport = TransportImpl::tmux(batch_settings);
let context = StartupContext {
namespace: task.bundle.bundle_name.clone(),
runtime_directory: task.runtime_directory.clone(),
target_member: target_member.clone(),
choose: noop_tmux_chooser(),
};
let _ = transport.startup(context);
Ok(transport)
}
SessionType::Ui => Ok(TransportImpl::ui(build_ui_transport_services(key))),
SessionType::Pubsub => Ok(TransportImpl::Pubsub),
SessionType::Acp => Err(super::super::super::relay_error(
"internal_unexpected_failure",
"ACP target reached the non-bootstrap worker construction path",
Some(json!({ "target_session": task.target_session })),
)),
}
}
fn prepare_coder_write(
task: &AsyncDeliveryTask,
transport: &mut TransportImpl,
) -> Result<OutcomeFuture, RelayError> {
match task.payload_mode {
DeliveryPayloadMode::EnvelopeMessage => {
let target_member = resolve_target_member(task)?;
let message = build_delivery_message(task, target_member, now_rfc3339().as_str());
emit_envelope_metadata_inscription(&message, task.message_id.as_str());
Ok(transport.mailw(build_coder_envelope(task, message)))
}
DeliveryPayloadMode::RawInput => {
Ok(transport.raww(task.message.clone(), task.append_enter))
}
}
}
fn collect_outcome(
joined: Result<InflightOutcome, tokio::task::JoinError>,
pending: &std::sync::atomic::AtomicUsize,
) {
let (task, record_served, outcome) = match joined {
Ok(value) => value,
Err(_join_error) => {
super::super::async_worker::release_pending_slot(pending);
return;
}
};
let send_result = match outcome {
Some(outcome) => outcome_to_send_result(&task, outcome),
None => dropped_send_result(&task),
};
if record_served && send_result.outcome == SendOutcome::Delivered {
let _ = note_session_served_successfully(
task.runtime_directory.as_path(),
task.target_session.as_str(),
);
}
super::super::async_worker::complete_task_outcome(&task, Ok(send_result));
super::super::async_worker::release_pending_slot(pending);
}
async fn shutdown_drain(
transport: Option<&mut TransportImpl>,
inflight: &mut JoinSet<InflightOutcome>,
receiver: &mut UnboundedReceiver<AsyncDeliveryTask>,
pending: &std::sync::atomic::AtomicUsize,
) {
if let Some(transport) = transport {
transport.shutdown();
}
while let Some(joined) = inflight.join_next().await {
collect_outcome(joined, pending);
}
super::super::async_worker::drop_pending_async_tasks_on_shutdown(receiver, pending);
}
fn noop_tmux_chooser() -> Chooser {
Arc::new(|_choice: ChoiceToMake| ChoiceMade::Cancelled {
decided_by: String::new(),
reason_code: "choice_unsupported".to_string(),
reason: Some("tmux transport does not raise operator choices".to_string()),
})
}
fn build_coder_envelope(task: &AsyncDeliveryTask, message: DeliveryMessage) -> DeliveryEnvelope {
DeliveryEnvelope {
message_id: task.message_id.clone(),
message,
append_enter: true,
choice_decider_sessions: task.choice_decider_sessions.clone(),
quiet_window: task.quiescence.quiet_window,
quiescence_timeout: task.quiescence.quiescence_timeout,
}
}
fn build_acp_driver_services(
key: &AsyncWorkerKey,
bootstrap: &AcpWorkerBootstrap,
) -> AcpDriverServices {
let namespace = key.namespace.clone();
let runtime_directory = bootstrap.runtime_directory.clone();
let target_session = bootstrap.target_member.id.clone();
let choices_pending_max = bootstrap.choices_pending_max;
AcpDriverServices {
mirror_state: {
let namespace = namespace.clone();
let runtime_directory = runtime_directory.clone();
let target_session = target_session.clone();
Arc::new(move |state| {
set_worker_readiness(
namespace.as_str(),
runtime_directory.as_path(),
target_session.as_str(),
state,
);
})
},
publish_output: {
let namespace = namespace.clone();
let runtime_directory = runtime_directory.clone();
let target_session = target_session.clone();
Arc::new(move |output_view| {
install_acp_worker_output_view(
namespace.as_str(),
runtime_directory.as_path(),
target_session.as_str(),
output_view,
);
})
},
broadcast_ui: {
let namespace = namespace.clone();
let target_session = target_session.clone();
Arc::new(move |event_type: &str, payload| {
broadcast_event_to_bundle_ui(
namespace.as_str(),
&acp_respawn_stream_event(
event_type,
namespace.as_str(),
target_session.as_str(),
payload,
),
);
})
},
invalidate_choices: {
let namespace = namespace.clone();
let runtime_directory = runtime_directory.clone();
let target_session = target_session.clone();
Arc::new(move || {
let context = ChoiceEventContext {
runtime_directory: runtime_directory.clone(),
namespace: namespace.clone(),
authorized_ui_sessions: list_registered_ui_sessions_for_bundle(
namespace.as_str(),
),
};
if let Err(reason) =
invalidate_pending_for_respawn(&context, target_session.as_str())
{
emit_inscription(
"relay.acp.respawn.choice_invalidate_failed",
&json!({
"namespace": namespace,
"target_session": target_session,
"reason": reason,
}),
);
}
})
},
chooser: build_acp_chooser(namespace, runtime_directory, choices_pending_max),
}
}
fn build_ui_transport_services(key: &AsyncWorkerKey) -> UiTransportServices {
let namespace = key.namespace.clone();
let target_session = key.target_session.clone();
UiTransportServices {
broadcast_incoming: {
let namespace = namespace.clone();
let target_session = target_session.clone();
Arc::new(move |incoming: &UiIncomingMessage| {
let mut payload = json!({
"message_id": incoming.message_id,
"sender_session": incoming.sender_session,
"body": incoming.body,
"cc_sessions": if incoming.cc_sessions.is_empty() {
Value::Null
} else {
json!(incoming.cc_sessions)
},
});
if let Some(authenticated_identity) = &incoming.authenticated_identity {
payload["authenticated_identity"] =
Value::String(authenticated_identity.clone());
}
let event = RelayStreamEvent {
event_type: "incoming_message".to_string(),
target_session: canonical_session_id(
target_session.as_str(),
namespace.as_str(),
),
created_at: now_rfc3339(),
payload,
};
stream_send_to_broadcast_status(send_event_to_registered_ui(
namespace.as_str(),
target_session.as_str(),
&event,
))
})
},
emit_phase: {
let namespace = namespace.clone();
let target_session = target_session.clone();
Arc::new(move |phase: UiOutcomePhase| {
let mut payload = serde_json::Map::new();
payload.insert("message_id".to_string(), Value::String(phase.message_id));
payload.insert("phase".to_string(), Value::String(phase.phase.to_string()));
payload.insert(
"outcome".to_string(),
phase
.outcome
.map(|value| Value::String(value.to_string()))
.unwrap_or(Value::Null),
);
if let Some(reason_code) = phase.reason_code {
payload.insert("reason_code".to_string(), Value::String(reason_code));
}
if let Some(reason) = phase.reason {
payload.insert("reason".to_string(), Value::String(reason));
}
let event = RelayStreamEvent {
event_type: "delivery_outcome".to_string(),
target_session: canonical_session_id(
target_session.as_str(),
namespace.as_str(),
),
created_at: now_rfc3339(),
payload: Value::Object(payload),
};
stream_send_to_broadcast_status(send_event_to_registered_ui(
namespace.as_str(),
target_session.as_str(),
&event,
))
})
},
}
}
fn stream_send_to_broadcast_status(
result: Result<StreamEventSendOutcome, std::io::Error>,
) -> UiBroadcastStatus {
match result {
Ok(StreamEventSendOutcome::Delivered) => UiBroadcastStatus::Delivered,
Ok(StreamEventSendOutcome::NoUiEndpoint | StreamEventSendOutcome::Disconnected) => {
UiBroadcastStatus::NoUi
}
Err(source) => {
UiBroadcastStatus::Failed(format!("failed to emit relay stream event: {source}"))
}
}
}
fn build_ui_envelope(task: &AsyncDeliveryTask) -> DeliveryEnvelope {
let target_member = task
.bundle
.members
.iter()
.find(|member| member.id == task.target_session);
let message = build_delivery_message(task, target_member, now_rfc3339().as_str());
DeliveryEnvelope {
message_id: task.message_id.clone(),
message,
append_enter: task.append_enter,
choice_decider_sessions: task.choice_decider_sessions.clone(),
quiet_window: task.quiescence.quiet_window,
quiescence_timeout: Some(
task.quiescence
.quiescence_timeout
.unwrap_or(Duration::from_millis(QUIESCENCE_TIMEOUT_MS_DEFAULT)),
),
}
}
fn outcome_to_send_result(task: &AsyncDeliveryTask, outcome: SingleDeliveryOutcome) -> SendResult {
SendResult {
target_session: task.target_session.clone(),
message_id: task.message_id.clone(),
outcome: outcome.outcome,
reason_code: outcome.reason_code,
reason: outcome.reason,
details: outcome.details,
}
}
fn dropped_send_result(task: &AsyncDeliveryTask) -> SendResult {
SendResult {
target_session: task.target_session.clone(),
message_id: task.message_id.clone(),
outcome: SendOutcome::DroppedOnShutdown,
reason_code: Some("dropped_on_shutdown".to_string()),
reason: Some("delivery worker dropped before completion".to_string()),
details: None,
}
}
fn now_rfc3339() -> String {
time::OffsetDateTime::now_utc()
.format(&Rfc3339)
.unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_string())
}
fn acp_respawn_stream_event(
event_type: &str,
namespace: &str,
target_session: &str,
payload: serde_json::Value,
) -> RelayStreamEvent {
RelayStreamEvent {
event_type: event_type.to_string(),
target_session: canonical_session_id(target_session, namespace),
created_at: time::OffsetDateTime::now_utc()
.format(&Rfc3339)
.unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_string()),
payload,
}
}