use std::{
collections::{HashMap, HashSet},
io,
path::Path,
sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard},
time::Duration,
};
use serde_json::{Value, json};
use time::OffsetDateTime;
use tokio::{
io::{AsyncBufReadExt, BufReader},
net::{UnixStream, unix::OwnedReadHalf},
sync::Notify,
time::error::Elapsed,
};
use crate::{
configuration::{
SessionType, load_bundle_configuration, load_policy_ids, load_tui_configuration,
},
runtime::{
inscriptions::emit_inscription,
paths::{BundleRuntimePaths, principal_store_path},
},
};
use super::drain::ConnectionWorkerSlot;
use super::identity::{
IdentityIntrospectRights, PrincipalStore, PrincipalType, VerifiedIdentity, split_principal_id,
verify_hello_credential,
};
use super::stream::{
HelloFrame, IncomingFrame, OutgoingFrame, RegisterStreamOutcome, SharedStreamWriter,
StreamRegistration, StreamRevokeSignal, parse_incoming_frame, register_stream,
registration_is_current, spawn_stream_writer, unregister_stream, write_stream_frame_to_writer,
};
use super::{
RelayError, RelayRequest, RelayResponse, RequestPrincipal, SCHEMA_VERSION,
canonical_session_id, dispatch_identity_admin, dispatch_identity_introspect, dispatch_list,
dispatch_look, dispatch_raww, dispatch_request, dispatch_send, handlers, map_config,
map_tui_config, relay_error,
};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum HostingIntent {
Run,
Hold,
}
struct CatalogEntry {
paths: BundleRuntimePaths,
hosting_intent: HostingIntent,
}
#[derive(Clone, Default)]
pub struct BundleCatalog {
bundles: Arc<RwLock<HashMap<String, CatalogEntry>>>,
}
impl BundleCatalog {
pub fn from_paths(paths: impl IntoIterator<Item = BundleRuntimePaths>) -> Self {
Self::from_entries(paths.into_iter().map(|paths| (paths, HostingIntent::Run)))
}
pub fn from_entries(
entries: impl IntoIterator<Item = (BundleRuntimePaths, HostingIntent)>,
) -> Self {
let bundles = entries
.into_iter()
.map(|(paths, hosting_intent)| {
(
paths.bundle_name.clone(),
CatalogEntry {
paths,
hosting_intent,
},
)
})
.collect();
Self {
bundles: Arc::new(RwLock::new(bundles)),
}
}
pub(super) fn lookup(&self, bundle_name: &str) -> Option<BundleRuntimePaths> {
self.read()
.get(bundle_name)
.map(|entry| entry.paths.clone())
}
pub fn snapshot(&self) -> Vec<BundleRuntimePaths> {
self.read()
.values()
.map(|entry| entry.paths.clone())
.collect()
}
pub(super) fn loaded_bundle_names(&self) -> HashSet<String> {
self.read().keys().cloned().collect()
}
pub(super) fn insert(&self, paths: BundleRuntimePaths, hosting_intent: HostingIntent) {
let bundle_name = paths.bundle_name.clone();
self.write().insert(
bundle_name,
CatalogEntry {
paths,
hosting_intent,
},
);
}
pub(super) fn remove(&self, bundle_name: &str) -> Option<BundleRuntimePaths> {
self.write().remove(bundle_name).map(|entry| entry.paths)
}
pub(super) fn set_intent(&self, bundle_name: &str, hosting_intent: HostingIntent) {
if let Some(entry) = self.write().get_mut(bundle_name) {
entry.hosting_intent = hosting_intent;
}
}
pub(super) fn is_held(&self, bundle_name: &str) -> bool {
self.read()
.get(bundle_name)
.is_some_and(|entry| entry.hosting_intent == HostingIntent::Hold)
}
fn read(&self) -> RwLockReadGuard<'_, HashMap<String, CatalogEntry>> {
self.bundles
.read()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
fn write(&self) -> RwLockWriteGuard<'_, HashMap<String, CatalogEntry>> {
self.bundles
.write()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
}
pub async fn serve_connection(
stream: UnixStream,
configuration_root: &Path,
state_root: &Path,
bundle_catalog: &BundleCatalog,
require_session_credentials: bool,
pre_hello_idle_timeout: Duration,
mut worker_slot: ConnectionWorkerSlot,
) -> Result<(), io::Error> {
let (read_half, write_half) = stream.into_split();
let (writer, mut writer_handle) = spawn_stream_writer(write_half);
let reader = BufReader::new(read_half);
let mut guard = RegistrationGuard::default();
let revoke = Arc::new(Notify::new());
let outcome = {
let frames = serve_connection_frames(
reader,
&writer,
&mut guard,
configuration_root,
state_root,
bundle_catalog,
require_session_credentials,
pre_hello_idle_timeout,
revoke.clone(),
&mut worker_slot,
);
tokio::pin!(frames);
tokio::select! {
biased;
result = &mut frames => result,
_ = &mut writer_handle => {
emit_inscription("relay.connection.writer_exit_teardown", &json!({}));
Ok(())
}
_ = revoke.notified() => {
emit_inscription("relay.connection.identity_revoked_teardown", &json!({}));
Ok(())
}
}
};
drop(writer);
drop(guard);
if !writer_handle.is_finished() {
let _ = writer_handle.await;
}
outcome
}
#[derive(Default)]
struct RegistrationGuard {
registration: Option<StreamRegistration>,
}
impl RegistrationGuard {
fn set(&mut self, registration: StreamRegistration) {
self.registration = Some(registration);
}
fn current(&self) -> Option<&StreamRegistration> {
self.registration.as_ref()
}
}
impl Drop for RegistrationGuard {
fn drop(&mut self) {
if let Some(registration) = self.registration.take() {
let _ = unregister_stream(®istration);
}
}
}
struct HelloBinding {
session_type: SessionType,
principal_id: String,
bound_bundle: Option<BundleRuntimePaths>,
store_backed: bool,
introspect_rights: Option<IdentityIntrospectRights>,
}
#[allow(clippy::too_many_arguments)]
async fn serve_connection_frames(
mut reader: BufReader<OwnedReadHalf>,
writer: &SharedStreamWriter,
guard: &mut RegistrationGuard,
configuration_root: &Path,
state_root: &Path,
bundle_catalog: &BundleCatalog,
require_session_credentials: bool,
pre_hello_idle_timeout: Duration,
revoke: StreamRevokeSignal,
worker_slot: &mut ConnectionWorkerSlot,
) -> Result<(), io::Error> {
let configuration_root: Arc<Path> = Arc::from(configuration_root);
let state_root: Arc<Path> = Arc::from(state_root);
let mut bound_bundle: Option<BundleRuntimePaths> = None;
let mut authenticated_identity: Option<String> = None;
let mut introspect_rights: Option<IdentityIntrospectRights> = None;
let mut line = String::new();
loop {
line.clear();
if worker_slot.shutdown_signaled() {
break;
}
let read = match read_next_line(
&mut reader,
&mut line,
guard.current().is_some(),
pre_hello_idle_timeout,
worker_slot,
)
.await
{
ReadLineOutcome::Read(read) => read,
ReadLineOutcome::Eof => break,
ReadLineOutcome::PreHelloIdleTimeout => break,
ReadLineOutcome::ShutdownRequested => break,
ReadLineOutcome::Error(source) => return Err(source),
};
if read == 0 {
break;
}
let trimmed = line.trim_end();
let frame = match parse_incoming_frame(trimmed) {
Ok(frame) => frame,
Err(source) => {
let response = RelayResponse::Error {
error: relay_error(
"validation_invalid_arguments",
"failed to parse relay request",
Some(json!({"cause": source.to_string()})),
),
};
write_stream_frame_to_writer(
writer,
OutgoingFrame::Response {
request_id: None,
response: &response,
},
)?;
break;
}
};
let _serving = worker_slot.begin_serving();
match frame {
IncomingFrame::Hello(hello) => {
let binding = match resolve_hello_binding(
&configuration_root,
&state_root,
bundle_catalog,
require_session_credentials,
&hello,
) {
Ok(binding) => binding,
Err(error) => {
write_stream_frame_to_writer(
writer,
OutgoingFrame::Response {
request_id: None,
response: &RelayResponse::Error { error },
},
)?;
break;
}
};
let connection_identity = binding.store_backed.then(|| hello.principal_id.clone());
let connection_scope = binding
.introspect_rights
.as_ref()
.and_then(|rights| rights.scope.clone());
match register_stream(
binding.principal_id.as_str(),
binding.session_type,
writer.clone(),
connection_identity.clone(),
revoke.clone(),
connection_scope,
)? {
RegisterStreamOutcome::Registered(value) => {
guard.set(value);
}
RegisterStreamOutcome::IdentityClaimConflict {
existing_connection_id,
} => {
let error = identity_claim_conflict_error(&hello, existing_connection_id);
write_stream_frame_to_writer(
writer,
OutgoingFrame::Response {
request_id: None,
response: &RelayResponse::Error { error },
},
)?;
break;
}
}
write_stream_frame_to_writer(
writer,
OutgoingFrame::HelloAck {
schema_version: SCHEMA_VERSION,
principal_id: hello.principal_id.as_str(),
},
)?;
if binding.session_type == SessionType::Ui
&& let Err(error) = emit_registration_choices_snapshots(
&configuration_root,
bundle_catalog,
&binding,
)
{
write_stream_frame_to_writer(
writer,
OutgoingFrame::Response {
request_id: None,
response: &RelayResponse::Error { error },
},
)?;
break;
}
authenticated_identity = connection_identity;
introspect_rights = binding.introspect_rights;
bound_bundle = binding.bound_bundle;
if let Some(rights) = introspect_rights.as_ref() {
match handlers::build_identity_snapshot_event(
&state_root,
hello.principal_id.as_str(),
rights,
) {
Ok(event) => write_stream_frame_to_writer(
writer,
OutgoingFrame::Event { event: &event },
)?,
Err(error) => {
write_stream_frame_to_writer(
writer,
OutgoingFrame::Response {
request_id: None,
response: &RelayResponse::Error { error },
},
)?;
break;
}
}
}
}
IncomingFrame::Request {
request_id,
namespace: target_namespace,
request,
} => {
let Some(active_registration) = guard.current() else {
let error = relay_error(
"validation_missing_hello",
"stream request requires hello registration",
None,
);
write_stream_frame_to_writer(
writer,
OutgoingFrame::Response {
request_id: request_id.as_deref(),
response: &RelayResponse::Error { error },
},
)?;
continue;
};
if !registration_is_current(active_registration)? {
let error = relay_error(
"validation_stale_stream_binding",
"stream binding has been replaced by a newer hello registration",
Some(json!({
"principal_id": active_registration.requester_session_id(),
"namespace": active_registration.namespace(),
})),
);
write_stream_frame_to_writer(
writer,
OutgoingFrame::Response {
request_id: request_id.as_deref(),
response: &RelayResponse::Error { error },
},
)?;
break;
}
if matches!(
request,
RelayRequest::NewPeer { .. } | RelayRequest::ChangePsk { .. }
) {
let requester_principal_id = full_requester_principal_id(active_registration);
let response = {
let configuration_root = Arc::clone(&configuration_root);
let state_root = Arc::clone(&state_root);
dispatch_on_blocking_pool(move || {
dispatch_identity_admin(
request,
&configuration_root,
&state_root,
requester_principal_id.as_str(),
)
})
.await
};
write_stream_frame_to_writer(
writer,
OutgoingFrame::Response {
request_id: request_id.as_deref(),
response: &response,
},
)?;
continue;
}
if matches!(request, RelayRequest::IdentityIntrospect { .. }) {
let principal = RequestPrincipal {
session_id: active_registration.requester_session_id().to_string(),
authenticated_identity: authenticated_identity.clone(),
introspect_rights: introspect_rights.clone(),
};
let response = {
let state_root = Arc::clone(&state_root);
dispatch_on_blocking_pool(move || {
dispatch_identity_introspect(request, &state_root, &principal)
})
.await
};
write_stream_frame_to_writer(
writer,
OutgoingFrame::Response {
request_id: request_id.as_deref(),
response: &response,
},
)?;
continue;
}
if matches!(request, RelayRequest::List { .. })
&& target_namespace.as_deref() == Some("GLOBAL")
{
let response = handlers::handle_global_list();
write_stream_frame_to_writer(
writer,
OutgoingFrame::Response {
request_id: request_id.as_deref(),
response: &response,
},
)?;
continue;
}
if matches!(request, RelayRequest::List { .. }) {
let enumerate_paths = match resolve_namespace_routing_bundle(
bundle_catalog,
target_namespace.as_deref(),
bound_bundle.as_ref(),
) {
Ok(paths) => paths,
Err(error) => {
write_stream_frame_to_writer(
writer,
OutgoingFrame::Response {
request_id: request_id.as_deref(),
response: &RelayResponse::Error { error },
},
)?;
continue;
}
};
let dispatch_paths = bound_bundle
.clone()
.unwrap_or_else(|| enumerate_paths.clone());
let response = {
let configuration_root = Arc::clone(&configuration_root);
dispatch_on_blocking_pool(move || {
dispatch_list(
request,
&configuration_root,
&dispatch_paths,
&enumerate_paths,
)
})
.await
};
write_stream_frame_to_writer(
writer,
OutgoingFrame::Response {
request_id: request_id.as_deref(),
response: &response,
},
)?;
continue;
}
if matches!(request, RelayRequest::Send { .. }) {
let principal = RequestPrincipal {
session_id: active_registration.requester_session_id().to_string(),
authenticated_identity: authenticated_identity.clone(),
introspect_rights: introspect_rights.clone(),
};
let response = {
let configuration_root = Arc::clone(&configuration_root);
let bound_bundle = bound_bundle.clone();
let bundle_catalog = bundle_catalog.clone();
dispatch_on_blocking_pool(move || {
dispatch_send(
request,
&configuration_root,
bound_bundle.as_ref(),
Some(principal),
&bundle_catalog,
)
})
.await
};
write_stream_frame_to_writer(
writer,
OutgoingFrame::Response {
request_id: request_id.as_deref(),
response: &response,
},
)?;
continue;
}
if matches!(request, RelayRequest::Look { .. }) {
let principal = RequestPrincipal {
session_id: active_registration.requester_session_id().to_string(),
authenticated_identity: authenticated_identity.clone(),
introspect_rights: introspect_rights.clone(),
};
let response = {
let configuration_root = Arc::clone(&configuration_root);
let bound_bundle = bound_bundle.clone();
let bundle_catalog = bundle_catalog.clone();
dispatch_on_blocking_pool(move || {
dispatch_look(
request,
&configuration_root,
bound_bundle.as_ref(),
Some(principal),
&bundle_catalog,
)
})
.await
};
write_stream_frame_to_writer(
writer,
OutgoingFrame::Response {
request_id: request_id.as_deref(),
response: &response,
},
)?;
continue;
}
if matches!(request, RelayRequest::Raww { .. }) {
let response = {
let configuration_root = Arc::clone(&configuration_root);
let bound_bundle = bound_bundle.clone();
let bundle_catalog = bundle_catalog.clone();
dispatch_on_blocking_pool(move || {
dispatch_raww(
request,
&configuration_root,
bound_bundle.as_ref(),
&bundle_catalog,
)
})
.await
};
write_stream_frame_to_writer(
writer,
OutgoingFrame::Response {
request_id: request_id.as_deref(),
response: &response,
},
)?;
continue;
}
let bundle_paths = match resolve_namespace_routing_bundle(
bundle_catalog,
target_namespace.as_deref(),
bound_bundle.as_ref(),
) {
Ok(bundle_paths) => bundle_paths,
Err(error) => {
write_stream_frame_to_writer(
writer,
OutgoingFrame::Response {
request_id: request_id.as_deref(),
response: &RelayResponse::Error { error },
},
)?;
continue;
}
};
let principal = RequestPrincipal {
session_id: active_registration.requester_session_id().to_string(),
authenticated_identity: authenticated_identity.clone(),
introspect_rights: introspect_rights.clone(),
};
let response = {
let configuration_root = Arc::clone(&configuration_root);
let bundle_catalog = bundle_catalog.clone();
dispatch_on_blocking_pool(move || {
dispatch_request(
request,
&configuration_root,
&bundle_paths.bundle_name,
&bundle_paths.runtime_directory,
Some(principal),
&bundle_catalog,
)
})
.await
};
write_stream_frame_to_writer(
writer,
OutgoingFrame::Response {
request_id: request_id.as_deref(),
response: &response,
},
)?;
}
}
}
Ok(())
}
async fn dispatch_on_blocking_pool(
dispatch: impl FnOnce() -> RelayResponse + Send + 'static,
) -> RelayResponse {
match tokio::task::spawn_blocking(dispatch).await {
Ok(response) => response,
Err(join_error) => RelayResponse::Error {
error: relay_error(
"internal_unexpected_failure",
"relay request dispatch task failed to join",
Some(json!({"cause": join_error.to_string()})),
),
},
}
}
enum ReadLineOutcome {
Read(usize),
Eof,
PreHelloIdleTimeout,
ShutdownRequested,
Error(io::Error),
}
async fn read_next_line(
reader: &mut BufReader<OwnedReadHalf>,
line: &mut String,
after_hello: bool,
pre_hello_idle_timeout: Duration,
worker_slot: &mut ConnectionWorkerSlot,
) -> ReadLineOutcome {
let read_result = if after_hello {
tokio::select! {
biased;
() = worker_slot.shutdown_signal() => return ReadLineOutcome::ShutdownRequested,
result = reader.read_line(line) => result,
}
} else {
tokio::select! {
biased;
() = worker_slot.shutdown_signal() => return ReadLineOutcome::ShutdownRequested,
result = tokio::time::timeout(pre_hello_idle_timeout, reader.read_line(line)) => {
match result {
Ok(result) => result,
Err(Elapsed { .. }) => return ReadLineOutcome::PreHelloIdleTimeout,
}
}
}
};
match read_result {
Ok(0) => ReadLineOutcome::Eof,
Ok(read) => ReadLineOutcome::Read(read),
Err(source) => ReadLineOutcome::Error(source),
}
}
fn full_requester_principal_id(registration: &StreamRegistration) -> String {
match registration.namespace() {
Some(namespace) => canonical_session_id(registration.requester_session_id(), namespace),
None => registration.requester_session_id().to_string(),
}
}
fn resolve_namespace_routing_bundle(
bundle_catalog: &BundleCatalog,
namespace: Option<&str>,
bound_bundle: Option<&BundleRuntimePaths>,
) -> Result<BundleRuntimePaths, RelayError> {
if let Some(namespace) = namespace {
return match namespace {
"EXTERNAL" | "RELAY" => Err(relay_error(
"validation_unsupported_namespace",
"namespace is reserved for relay-internal routing and cannot be selected by a client",
Some(json!({ "namespace": namespace })),
)),
bundle_name => bundle_catalog
.lookup(bundle_name)
.ok_or_else(|| unknown_bundle_error(bundle_name)),
};
}
if let Some(bound) = bound_bundle {
return Ok(bound.clone());
}
Err(relay_error(
"validation_missing_routing_namespace",
"stream request from a relay-wide principal requires an explicit routing namespace",
None,
))
}
fn unknown_bundle_error(bundle_name: &str) -> RelayError {
relay_error(
"validation_unknown_bundle",
"request target bundle is not configured on this relay",
Some(json!({ "bundle_name": bundle_name })),
)
}
fn identity_claim_conflict_error(
hello: &HelloFrame,
existing_connection_id: Option<String>,
) -> RelayError {
let mut details = serde_json::Map::new();
details.insert(
"principal_id".to_string(),
Value::String(hello.principal_id.clone()),
);
details.insert(
"reason".to_string(),
Value::String("existing identity owner is still live".to_string()),
);
if let Some(value) = existing_connection_id {
details.insert("existing_connection_id".to_string(), Value::String(value));
}
relay_error(
"runtime_identity_claim_conflict",
"stream identity is already claimed by a live connection",
Some(Value::Object(details)),
)
}
fn emit_registration_choices_snapshots(
configuration_root: &Path,
bundle_catalog: &BundleCatalog,
binding: &HelloBinding,
) -> Result<(), RelayError> {
match binding.bound_bundle.as_ref() {
Some(bundle_paths) => {
if let Some((session_id, namespace)) = split_principal_id(binding.principal_id.as_str())
{
handlers::emit_choices_snapshot_for_ui_registration(
configuration_root,
namespace,
&bundle_paths.runtime_directory,
session_id,
)?;
}
}
None => {
for bundle_paths in bundle_catalog.snapshot() {
handlers::emit_choices_snapshot_for_ui_registration(
configuration_root,
&bundle_paths.bundle_name,
&bundle_paths.runtime_directory,
binding.principal_id.as_str(),
)?;
}
}
}
Ok(())
}
fn resolve_hello_binding(
configuration_root: &Path,
state_root: &Path,
bundle_catalog: &BundleCatalog,
require_session_credentials: bool,
hello: &HelloFrame,
) -> Result<HelloBinding, RelayError> {
if hello.schema_version != SCHEMA_VERSION {
return Err(relay_error(
"validation_invalid_schema_version",
"hello schema_version is not supported",
Some(json!({
"schema_version": hello.schema_version,
"supported_schema_version": SCHEMA_VERSION,
})),
));
}
let store = PrincipalStore::load(principal_store_path(state_root))?;
let verified = verify_hello_credential(
hello.principal_id.as_str(),
hello.identity_token.as_str(),
&store,
require_session_credentials,
OffsetDateTime::now_utc(),
)?;
let VerifiedIdentity {
principal_type,
store_backed,
introspect_rights,
} = verified;
match principal_type {
PrincipalType::Session => {
let (session_id, namespace) = split_principal_id(hello.principal_id.as_str())
.ok_or_else(|| {
relay_error(
"validation_invalid_principal_id",
"session principal_id is not in <session>@<bundle> form",
Some(json!({ "principal_id": hello.principal_id })),
)
})?;
let bundle_paths = bundle_catalog
.lookup(namespace)
.ok_or_else(|| unknown_bundle_error(namespace))?;
let session_type =
resolve_bundle_member_session_type(configuration_root, namespace, session_id)?;
Ok(HelloBinding {
session_type,
principal_id: hello.principal_id.clone(),
bound_bundle: Some(bundle_paths),
store_backed,
introspect_rights,
})
}
PrincipalType::User => {
let session_type =
resolve_global_user_session_type(configuration_root, hello.principal_id.as_str())?;
Ok(HelloBinding {
session_type,
principal_id: hello.principal_id.clone(),
bound_bundle: None,
store_backed,
introspect_rights,
})
}
PrincipalType::Application | PrincipalType::Relay => Ok(HelloBinding {
session_type: SessionType::Pubsub,
principal_id: hello.principal_id.clone(),
bound_bundle: None,
store_backed,
introspect_rights,
}),
}
}
fn resolve_bundle_member_session_type(
configuration_root: &Path,
bundle_name: &str,
session_id: &str,
) -> Result<SessionType, RelayError> {
let bundle = load_bundle_configuration(configuration_root, bundle_name).map_err(map_config)?;
let Some(member) = bundle.members.iter().find(|member| member.id == session_id) else {
return Err(relay_error(
"validation_unknown_sender",
"hello session_id is not configured in associated bundle",
Some(json!({
"bundle_name": bundle.bundle_name,
"session_id": session_id,
})),
));
};
Ok(member.target.session_type())
}
fn resolve_global_user_session_type(
configuration_root: &Path,
principal_id: &str,
) -> Result<SessionType, RelayError> {
let Some(users_configuration) =
load_tui_configuration(configuration_root).map_err(map_tui_config)?
else {
return Err(global_user_missing_error(principal_id));
};
let Some(user_session) = users_configuration.session_by_id(principal_id) else {
return Err(global_user_missing_error(principal_id));
};
let policy_ids = load_policy_ids(configuration_root).map_err(map_tui_config)?;
if !policy_ids.contains(user_session.policy.as_str()) {
return Err(relay_error(
"validation_unknown_policy",
"global user policy references unknown policy id",
Some(json!({
"session_id": user_session.id,
"policy_id": user_session.policy,
})),
));
}
Ok(user_session.session_type)
}
fn global_user_missing_error(principal_id: &str) -> RelayError {
relay_error(
"validation_unknown_sender",
"hello principal_id is not configured in global users",
Some(json!({ "principal_id": principal_id })),
)
}