use std::{
collections::HashMap,
io,
sync::{Arc, Mutex, OnceLock},
time::Duration,
};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use tokio::{
io::AsyncWriteExt,
net::unix::OwnedWriteHalf,
sync::{
Notify,
mpsc::{self, UnboundedSender, error::SendError},
},
task::JoinHandle,
time::timeout,
};
use uuid::Uuid;
use crate::configuration::SessionType;
use crate::runtime::inscriptions::emit_inscription;
use super::identity::{
PrincipalType, canonical_session_id, classify_principal_id, scope_permits, split_principal_id,
};
use super::{RelayRequest, RelayResponse};
const RELAY_CONNECTION_WRITE_TIMEOUT: Duration = Duration::from_secs(5);
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
pub(super) struct HelloFrame {
pub(super) schema_version: String,
pub(super) principal_id: String,
pub(super) identity_token: String,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum RegistrationSource {
Configured,
Stream,
}
#[derive(Clone, Debug)]
pub(super) struct StreamRegistration {
pub(super) principal_id: String,
bound_namespace: Option<String>,
requester_session: String,
pub(super) stream_id: String,
}
impl StreamRegistration {
pub(super) fn requester_session_id(&self) -> &str {
self.requester_session.as_str()
}
pub(super) fn namespace(&self) -> Option<&str> {
self.bound_namespace.as_deref()
}
}
pub(super) type SharedStreamWriter = UnboundedSender<Vec<u8>>;
pub(super) type StreamRevokeSignal = Arc<Notify>;
#[derive(Clone, Debug, PartialEq)]
pub(super) enum IncomingFrame {
Hello(HelloFrame),
Request {
request_id: Option<String>,
namespace: Option<String>,
request: RelayRequest,
},
}
#[derive(Clone, Debug, Deserialize)]
#[serde(tag = "frame", rename_all = "snake_case")]
enum IncomingEnvelope {
Hello {
schema_version: String,
principal_id: String,
identity_token: String,
},
Request {
#[serde(default)]
request_id: Option<String>,
#[serde(default)]
namespace: Option<String>,
request: RelayRequest,
},
}
#[derive(Clone, Debug, Serialize)]
#[serde(tag = "frame", rename_all = "snake_case")]
pub(super) enum OutgoingFrame<'a> {
HelloAck {
schema_version: &'a str,
principal_id: &'a str,
},
Response {
#[serde(skip_serializing_if = "Option::is_none")]
request_id: Option<&'a str>,
response: &'a RelayResponse,
},
Event {
event: &'a RelayStreamEvent,
},
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
pub(super) struct RelayStreamEvent {
pub(super) event_type: String,
pub(super) target_session: String,
pub(super) created_at: String,
pub(super) payload: Value,
}
#[derive(Clone, Debug)]
struct RegistryEntry {
principal_id: String,
session_id: String,
namespace: String,
principal_class: PrincipalType,
source: RegistrationSource,
session_type: SessionType,
stream_id: Option<String>,
writer: Option<SharedStreamWriter>,
authenticated_identity: Option<String>,
revoke: Option<StreamRevokeSignal>,
scope: Option<String>,
}
impl RegistryEntry {
fn is_connected(&self) -> bool {
self.stream_id.is_some()
&& self
.writer
.as_ref()
.is_some_and(|writer| !writer.is_closed())
}
fn clear_dynamic_state(&mut self) {
self.stream_id = None;
self.writer = None;
self.revoke = None;
self.authenticated_identity = None;
self.scope = None;
}
}
#[derive(Default)]
struct StreamRegistry {
entries: Mutex<HashMap<String, RegistryEntry>>,
}
static STREAM_REGISTRY: OnceLock<StreamRegistry> = OnceLock::new();
fn parse_principal_parts(principal_id: &str) -> Option<(String, String, PrincipalType)> {
let class = classify_principal_id(principal_id)?;
let (session_id, namespace) = split_principal_id(principal_id)?;
Some((session_id.to_string(), namespace.to_string(), class))
}
pub(super) fn parse_incoming_frame(line: &str) -> Result<IncomingFrame, io::Error> {
let envelope = serde_json::from_str::<IncomingEnvelope>(line).map_err(io::Error::other)?;
match envelope {
IncomingEnvelope::Hello {
schema_version,
principal_id,
identity_token,
} => Ok(IncomingFrame::Hello(HelloFrame {
schema_version,
principal_id,
identity_token,
})),
IncomingEnvelope::Request {
request_id,
namespace,
request,
} => Ok(IncomingFrame::Request {
request_id,
namespace,
request,
}),
}
}
pub(super) fn encode_outgoing_frame(frame: OutgoingFrame<'_>) -> Result<String, io::Error> {
serde_json::to_string(&frame).map_err(io::Error::other)
}
pub(super) fn spawn_stream_writer(
mut write_half: OwnedWriteHalf,
) -> (SharedStreamWriter, JoinHandle<()>) {
let (sender, mut receiver) = mpsc::unbounded_channel::<Vec<u8>>();
let handle = tokio::spawn(async move {
while let Some(bytes) = receiver.recv().await {
let write = async {
write_half.write_all(&bytes).await?;
write_half.flush().await
};
match timeout(relay_connection_write_timeout(), write).await {
Ok(Ok(())) => {}
Ok(Err(source)) => {
note_write_timeout(&source);
break;
}
Err(_elapsed) => {
let timeout_error =
io::Error::new(io::ErrorKind::TimedOut, "relay connection write timed out");
note_write_timeout(&timeout_error);
break;
}
}
}
});
(sender, handle)
}
fn relay_connection_write_timeout() -> Duration {
std::env::var("AGENTMUX_RELAY_CONNECTION_WRITE_TIMEOUT_MS")
.ok()
.and_then(|value| value.parse::<u64>().ok())
.filter(|value| *value > 0)
.map(Duration::from_millis)
.unwrap_or(RELAY_CONNECTION_WRITE_TIMEOUT)
}
pub(super) fn register_configured_session(
principal_id: &str,
session_type: SessionType,
) -> Result<(), io::Error> {
let (session_id, namespace, principal_class) = parse_principal_parts(principal_id)
.ok_or_else(|| io::Error::other("configured principal_id is not qualified"))?;
let registry = stream_registry();
let mut entries = registry
.entries
.lock()
.map_err(|_| io::Error::other("failed to lock stream registry"))?;
match entries.get_mut(principal_id) {
Some(entry) => {
entry.source = RegistrationSource::Configured;
entry.session_type = session_type;
entry.namespace = namespace;
entry.session_id = session_id;
entry.principal_class = principal_class;
}
None => {
entries.insert(
principal_id.to_string(),
RegistryEntry {
principal_id: principal_id.to_string(),
session_id,
namespace,
principal_class,
source: RegistrationSource::Configured,
session_type,
stream_id: None,
writer: None,
authenticated_identity: None,
revoke: None,
scope: None,
},
);
}
}
Ok(())
}
pub(super) fn register_stream(
principal_id: &str,
session_type: SessionType,
writer: SharedStreamWriter,
authenticated_identity: Option<String>,
revoke: StreamRevokeSignal,
scope: Option<String>,
) -> Result<RegisterStreamOutcome, io::Error> {
let (session_id, namespace, principal_class) = parse_principal_parts(principal_id)
.ok_or_else(|| io::Error::other("hello principal_id is not a qualified principal"))?;
let registry = stream_registry();
let mut entries = registry
.entries
.lock()
.map_err(|_| io::Error::other("failed to lock stream registry"))?;
if let Some(entry) = entries.get_mut(principal_id) {
if entry.is_connected() {
return Ok(RegisterStreamOutcome::IdentityClaimConflict {
existing_connection_id: entry.stream_id.clone(),
});
}
if entry.stream_id.is_some() {
let prior_stream_id = entry.stream_id.clone();
entry.clear_dynamic_state();
emit_inscription(
"relay.stream.stale_claim_reclaimed",
&serde_json::json!({
"principal_id": principal_id,
"prior_stream_id": prior_stream_id,
}),
);
}
}
let stream_id = Uuid::new_v4().to_string();
match entries.get_mut(principal_id) {
Some(entry) => {
entry.stream_id = Some(stream_id.clone());
entry.session_type = session_type;
entry.writer = Some(writer);
entry.authenticated_identity = authenticated_identity;
entry.revoke = Some(revoke);
entry.scope = scope;
}
None => {
entries.insert(
principal_id.to_string(),
RegistryEntry {
principal_id: principal_id.to_string(),
session_id: session_id.clone(),
namespace: namespace.clone(),
principal_class,
source: RegistrationSource::Stream,
session_type,
stream_id: Some(stream_id.clone()),
writer: Some(writer),
authenticated_identity,
revoke: Some(revoke),
scope,
},
);
}
}
let bound_namespace = (principal_class == PrincipalType::Session).then(|| namespace.clone());
let requester_session = if principal_class == PrincipalType::Session {
session_id
} else {
principal_id.to_string()
};
Ok(RegisterStreamOutcome::Registered(StreamRegistration {
principal_id: principal_id.to_string(),
bound_namespace,
requester_session,
stream_id,
}))
}
#[derive(Clone, Debug)]
pub(super) enum RegisterStreamOutcome {
Registered(StreamRegistration),
IdentityClaimConflict {
existing_connection_id: Option<String>,
},
}
pub(super) fn registration_is_current(
registration: &StreamRegistration,
) -> Result<bool, io::Error> {
let registry = stream_registry();
let entries = registry
.entries
.lock()
.map_err(|_| io::Error::other("failed to lock stream registry"))?;
Ok(entries
.get(registration.principal_id.as_str())
.is_some_and(|entry| entry.stream_id.as_deref() == Some(registration.stream_id.as_str())))
}
pub(super) fn unregister_stream(registration: &StreamRegistration) -> Result<(), io::Error> {
let registry = stream_registry();
let mut entries = registry
.entries
.lock()
.map_err(|_| io::Error::other("failed to lock stream registry"))?;
if let Some(entry) = entries.get_mut(registration.principal_id.as_str())
&& entry
.stream_id
.as_deref()
.is_some_and(|stream_id| stream_id == registration.stream_id.as_str())
{
if entry.source == RegistrationSource::Stream {
entries.remove(registration.principal_id.as_str());
} else {
entry.clear_dynamic_state();
}
}
Ok(())
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum EvictionScope {
DropStreamSourceOnly,
DropEntry,
}
fn evict_streams<F>(selector: F, response: &RelayResponse, removal: EvictionScope) -> usize
where
F: Fn(&RegistryEntry) -> bool,
{
let registry = stream_registry();
let targets = {
let Ok(mut entries) = registry.entries.lock() else {
return 0;
};
let mut targets = Vec::new();
let mut to_remove = Vec::new();
for (key, entry) in entries.iter_mut() {
if !selector(entry) {
continue;
}
if let (Some(writer), Some(revoke)) = (entry.writer.clone(), entry.revoke.clone()) {
entry.clear_dynamic_state();
targets.push((writer, revoke));
}
let remove = match removal {
EvictionScope::DropEntry => true,
EvictionScope::DropStreamSourceOnly => entry.source == RegistrationSource::Stream,
};
if remove {
to_remove.push(key.clone());
}
}
for key in to_remove {
entries.remove(&key);
}
targets
};
let count = targets.len();
for (writer, revoke) in targets {
let _ = write_stream_frame_to_writer(
&writer,
OutgoingFrame::Response {
request_id: None,
response,
},
);
revoke.notify_one();
}
count
}
pub(super) fn revoke_streams_for_identity(principal_id: &str, response: &RelayResponse) -> usize {
evict_streams(
|entry| entry.authenticated_identity.as_deref() == Some(principal_id),
response,
EvictionScope::DropStreamSourceOnly,
)
}
pub(super) fn evict_streams_for_bundle(namespace: &str, response: &RelayResponse) -> usize {
evict_streams(
|entry| entry.namespace == namespace,
response,
EvictionScope::DropEntry,
)
}
pub(super) fn notify_trusted_hosts_of_revocation(
revoked_principal_id: &str,
template: &RelayStreamEvent,
) -> usize {
let registry = stream_registry();
let targets = {
let Ok(entries) = registry.entries.lock() else {
return 0;
};
let mut targets = Vec::new();
for entry in entries.values() {
let Some(writer) = entry.writer.clone() else {
continue;
};
if !scope_permits(entry.scope.as_deref(), revoked_principal_id) {
continue;
}
if !is_relay_wide(entry.principal_class) {
continue;
}
targets.push((writer, entry.principal_id.clone()));
}
targets
};
let count = targets.len();
for (writer, host_principal_id) in targets {
let mut event = template.clone();
event.target_session = host_principal_id;
let _ = write_stream_frame_to_writer(&writer, OutgoingFrame::Event { event: &event });
}
count
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum StreamEventSendOutcome {
Delivered,
NoUiEndpoint,
Disconnected,
}
fn is_relay_wide(principal_class: PrincipalType) -> bool {
matches!(
principal_class,
PrincipalType::User | PrincipalType::Application | PrincipalType::Relay
)
}
pub(in crate::relay) fn registry_target_is_relay_wide(principal_id: &str) -> Option<bool> {
let registry = stream_registry();
let entries = registry.entries.lock().ok()?;
entries
.get(principal_id)
.map(|entry| is_relay_wide(entry.principal_class))
}
pub(super) fn lookup_registry_session_type(principal_id: &str) -> Option<SessionType> {
let registry = stream_registry();
let entries = registry.entries.lock().ok()?;
entries.get(principal_id).map(|entry| entry.session_type)
}
pub(super) fn list_registered_ui_sessions_for_bundle(namespace: &str) -> Vec<String> {
let registry = stream_registry();
let Ok(entries) = registry.entries.lock() else {
return Vec::new();
};
entries
.values()
.filter_map(|entry| {
if entry.session_type != SessionType::Ui {
return None;
}
entry.writer.as_ref()?;
if is_relay_wide(entry.principal_class) {
Some(entry.principal_id.clone())
} else if entry.namespace == namespace {
Some(entry.session_id.clone())
} else {
None
}
})
.collect()
}
pub(super) fn list_namespace_sessions(namespace: &str) -> Vec<(String, SessionType, bool)> {
let registry = stream_registry();
let Ok(entries) = registry.entries.lock() else {
return Vec::new();
};
entries
.values()
.filter(|entry| entry.namespace == namespace)
.map(|entry| {
(
entry.principal_id.clone(),
entry.session_type,
entry.is_connected(),
)
})
.collect()
}
pub(super) fn broadcast_event_to_bundle_ui(
namespace: &str,
template: &RelayStreamEvent,
) -> Vec<String> {
let ui_session_ids = list_registered_ui_sessions_for_bundle(namespace);
let mut delivered = Vec::new();
for ui_session_id in ui_session_ids {
let mut event = template.clone();
event.target_session = canonical_session_id(ui_session_id.as_str(), namespace);
if matches!(
send_event_to_registered_ui(namespace, ui_session_id.as_str(), &event),
Ok(StreamEventSendOutcome::Delivered)
) {
delivered.push(ui_session_id);
}
}
delivered
}
pub(super) fn send_event_to_registered_ui(
namespace: &str,
session_id: &str,
event: &RelayStreamEvent,
) -> Result<StreamEventSendOutcome, io::Error> {
let registry = stream_registry();
let principal_id = canonical_session_id(session_id, namespace);
let (session_type, writer) = {
let entries = registry
.entries
.lock()
.map_err(|_| io::Error::other("failed to lock stream registry"))?;
let Some(entry) = entries.get(principal_id.as_str()) else {
return Ok(StreamEventSendOutcome::NoUiEndpoint);
};
(entry.session_type, entry.writer.clone())
};
if session_type != SessionType::Ui {
return Ok(StreamEventSendOutcome::NoUiEndpoint);
}
let Some(writer) = writer else {
return Ok(StreamEventSendOutcome::Disconnected);
};
if write_stream_frame_to_writer(&writer, OutgoingFrame::Event { event }).is_ok() {
return Ok(StreamEventSendOutcome::Delivered);
}
let mut entries = registry
.entries
.lock()
.map_err(|_| io::Error::other("failed to lock stream registry"))?;
if let Some(entry) = entries.get_mut(principal_id.as_str()) {
entry.clear_dynamic_state();
}
Ok(StreamEventSendOutcome::Disconnected)
}
pub(super) fn write_stream_frame_to_writer(
writer: &SharedStreamWriter,
frame: OutgoingFrame<'_>,
) -> Result<(), io::Error> {
let mut bytes = encode_outgoing_frame(frame)?.into_bytes();
bytes.push(b'\n');
enqueue_bytes(writer, bytes)
}
fn enqueue_bytes(writer: &SharedStreamWriter, bytes: Vec<u8>) -> Result<(), io::Error> {
writer.send(bytes).map_err(|SendError(_)| {
io::Error::new(io::ErrorKind::BrokenPipe, "stream writer task has closed")
})
}
fn note_write_timeout(error: &io::Error) {
if matches!(
error.kind(),
io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut
) {
emit_inscription(
"relay.connection.write_timeout",
&serde_json::json!({ "cause": error.to_string() }),
);
}
}
fn stream_registry() -> &'static StreamRegistry {
STREAM_REGISTRY.get_or_init(StreamRegistry::default)
}
#[doc(hidden)]
#[must_use]
pub fn second_claim_is_live_conflict_for_testing(prior_writer_open: bool) -> bool {
let principal_id = format!("staleprobe{}@GLOBAL", Uuid::new_v4().simple());
let revoke: StreamRevokeSignal = Arc::new(Notify::new());
let (first_writer, first_receiver) = mpsc::unbounded_channel::<Vec<u8>>();
let _ = register_stream(
principal_id.as_str(),
SessionType::Ui,
first_writer,
None,
revoke.clone(),
None,
);
if !prior_writer_open {
drop(first_receiver);
}
let (second_writer, _second_receiver) = mpsc::unbounded_channel::<Vec<u8>>();
let outcome = register_stream(
principal_id.as_str(),
SessionType::Ui,
second_writer,
None,
revoke,
None,
);
if let Ok(mut entries) = stream_registry().entries.lock() {
entries.remove(principal_id.as_str());
}
matches!(
outcome,
Ok(RegisterStreamOutcome::IdentityClaimConflict { .. })
)
}