use std::collections::{BTreeMap, BTreeSet};
use std::sync::{Arc, Mutex};
use std::time::{SystemTime, UNIX_EPOCH};
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use crate::{
FrontendAttachment, FrontendOperationInvocation, FrontendOperationResult, FrontendResponse,
FrontendRuntimeDescriptor, SdkError, SdkOperation, SdkRuntime,
};
pub const DEFAULT_RUNTIME_LEASE_TTL_MS: u64 = 30_000;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RuntimePermission {
Observe,
Interact,
Approve,
Terminate,
}
impl RuntimePermission {
pub const fn as_str(self) -> &'static str {
match self {
Self::Observe => "observe",
Self::Interact => "interact",
Self::Approve => "approve",
Self::Terminate => "terminate",
}
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeAuthorization {
permissions: BTreeSet<RuntimePermission>,
}
impl RuntimeAuthorization {
pub fn new(permissions: impl IntoIterator<Item = RuntimePermission>) -> Self {
Self {
permissions: permissions.into_iter().collect(),
}
}
pub fn owner() -> Self {
Self::new([
RuntimePermission::Observe,
RuntimePermission::Interact,
RuntimePermission::Approve,
RuntimePermission::Terminate,
])
}
pub fn interactive() -> Self {
Self::new([
RuntimePermission::Observe,
RuntimePermission::Interact,
RuntimePermission::Approve,
])
}
pub fn observer() -> Self {
Self::new([RuntimePermission::Observe])
}
pub fn allows(&self, permission: RuntimePermission) -> bool {
self.permissions.contains(&permission)
}
pub fn permissions(&self) -> impl Iterator<Item = RuntimePermission> + '_ {
self.permissions.iter().copied()
}
pub fn restrict_to(&self, requested: &Self) -> Self {
Self::new(
self.permissions
.intersection(&requested.permissions)
.copied(),
)
}
pub fn header_value(&self) -> String {
self.permissions()
.map(RuntimePermission::as_str)
.collect::<Vec<_>>()
.join(",")
}
pub fn parse_header(value: &str) -> Result<Self, RuntimeLeaseError> {
if value.is_empty() {
return Ok(Self::default());
}
let mut permissions = Vec::new();
for name in value.split(',') {
let permission = match name {
"observe" => RuntimePermission::Observe,
"interact" => RuntimePermission::Interact,
"approve" => RuntimePermission::Approve,
"terminate" => RuntimePermission::Terminate,
_ => return Err(RuntimeLeaseError::InvalidAuthorization),
};
permissions.push(permission);
}
Ok(Self::new(permissions))
}
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(transparent)]
pub struct RuntimeClientId(String);
impl RuntimeClientId {
pub fn parse(value: impl Into<String>) -> Result<Self, RuntimeLeaseError> {
let value = value.into();
if value.is_empty()
|| value.len() > 128
|| !value
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.'))
{
return Err(RuntimeLeaseError::InvalidClientId);
}
Ok(Self(value))
}
pub fn as_str(&self) -> &str {
&self.0
}
}
impl std::fmt::Display for RuntimeClientId {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(&self.0)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeControllerLease {
pub client_id: RuntimeClientId,
pub expires_at_ms: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeObserverLease {
pub client_id: RuntimeClientId,
pub last_seen_ms: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeLeaseSnapshot {
pub controller: Option<RuntimeControllerLease>,
pub observers: Vec<RuntimeObserverLease>,
pub lease_ttl_ms: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum RuntimeLeaseError {
#[error("invalid runtime client id")]
InvalidClientId,
#[error("invalid runtime authorization grant")]
InvalidAuthorization,
#[error("runtime permission `{0:?}` is required")]
Unauthorized(RuntimePermission),
#[error("controller lease is held by `{holder}` until {expires_at_ms}")]
ControllerHeld {
holder: RuntimeClientId,
expires_at_ms: u64,
},
#[error("controller lease required")]
ControllerRequired,
#[error("controller lease expired")]
LeaseExpired,
}
#[derive(Debug)]
pub struct RuntimeLeaseCoordinator {
lease_ttl_ms: u64,
controller: Option<RuntimeControllerLease>,
observers: BTreeMap<RuntimeClientId, RuntimeObserverLease>,
expired_controller: Option<RuntimeClientId>,
}
impl RuntimeLeaseCoordinator {
pub fn new(lease_ttl_ms: u64) -> Self {
assert!(lease_ttl_ms > 0, "runtime lease TTL must be non-zero");
Self {
lease_ttl_ms,
controller: None,
observers: BTreeMap::new(),
expired_controller: None,
}
}
pub fn attach(
&mut self,
client_id: RuntimeClientId,
authorization: &RuntimeAuthorization,
now_ms: u64,
) -> Result<RuntimeLeaseSnapshot, RuntimeLeaseError> {
require(authorization, RuntimePermission::Observe)?;
self.reconcile(now_ms);
self.observers.insert(
client_id.clone(),
RuntimeObserverLease {
client_id,
last_seen_ms: now_ms,
},
);
Ok(self.snapshot(now_ms))
}
pub fn heartbeat(
&mut self,
client_id: &RuntimeClientId,
authorization: &RuntimeAuthorization,
now_ms: u64,
) -> Result<RuntimeLeaseSnapshot, RuntimeLeaseError> {
require(authorization, RuntimePermission::Observe)?;
self.reconcile(now_ms);
self.observers
.entry(client_id.clone())
.and_modify(|observer| observer.last_seen_ms = now_ms)
.or_insert_with(|| RuntimeObserverLease {
client_id: client_id.clone(),
last_seen_ms: now_ms,
});
Ok(self.snapshot(now_ms))
}
pub fn authorize(
&mut self,
authorization: &RuntimeAuthorization,
permission: RuntimePermission,
now_ms: u64,
) -> Result<RuntimeLeaseSnapshot, RuntimeLeaseError> {
require(authorization, permission)?;
Ok(self.snapshot(now_ms))
}
pub fn claim_control(
&mut self,
client_id: RuntimeClientId,
authorization: &RuntimeAuthorization,
now_ms: u64,
) -> Result<RuntimeLeaseSnapshot, RuntimeLeaseError> {
require(authorization, RuntimePermission::Interact)?;
self.attach(client_id.clone(), authorization, now_ms)?;
if self.controller.is_none() && self.expired_controller.as_ref() == Some(&client_id) {
return Err(RuntimeLeaseError::LeaseExpired);
}
match &self.controller {
Some(lease) if lease.client_id != client_id => {
return Err(RuntimeLeaseError::ControllerHeld {
holder: lease.client_id.clone(),
expires_at_ms: lease.expires_at_ms,
});
}
_ => {}
}
self.controller = Some(RuntimeControllerLease {
client_id,
expires_at_ms: now_ms.saturating_add(self.lease_ttl_ms),
});
self.expired_controller = None;
Ok(self.snapshot(now_ms))
}
pub fn take_control(
&mut self,
client_id: RuntimeClientId,
authorization: &RuntimeAuthorization,
now_ms: u64,
) -> Result<RuntimeLeaseSnapshot, RuntimeLeaseError> {
require(authorization, RuntimePermission::Interact)?;
self.attach(client_id.clone(), authorization, now_ms)?;
self.controller = Some(RuntimeControllerLease {
client_id,
expires_at_ms: now_ms.saturating_add(self.lease_ttl_ms),
});
self.expired_controller = None;
Ok(self.snapshot(now_ms))
}
pub fn authorize_controller(
&mut self,
client_id: &RuntimeClientId,
authorization: &RuntimeAuthorization,
permission: RuntimePermission,
now_ms: u64,
) -> Result<RuntimeLeaseSnapshot, RuntimeLeaseError> {
require(authorization, permission)?;
let was_expired = self
.controller
.as_ref()
.is_some_and(|lease| lease.client_id == *client_id && lease.expires_at_ms <= now_ms);
self.reconcile(now_ms);
let Some(controller) = &mut self.controller else {
return Err(
if was_expired || self.expired_controller.as_ref() == Some(client_id) {
RuntimeLeaseError::LeaseExpired
} else {
RuntimeLeaseError::ControllerRequired
},
);
};
if controller.client_id != *client_id {
return Err(RuntimeLeaseError::ControllerRequired);
}
controller.expires_at_ms = now_ms.saturating_add(self.lease_ttl_ms);
self.expired_controller = None;
if let Some(observer) = self.observers.get_mut(client_id) {
observer.last_seen_ms = now_ms;
}
Ok(self.snapshot(now_ms))
}
pub fn detach(&mut self, client_id: &RuntimeClientId, now_ms: u64) -> RuntimeLeaseSnapshot {
self.reconcile(now_ms);
self.observers.remove(client_id);
if self
.controller
.as_ref()
.is_some_and(|lease| lease.client_id == *client_id)
{
self.controller = None;
}
if self.expired_controller.as_ref() == Some(client_id) {
self.expired_controller = None;
}
self.snapshot(now_ms)
}
pub fn snapshot(&mut self, now_ms: u64) -> RuntimeLeaseSnapshot {
self.reconcile(now_ms);
RuntimeLeaseSnapshot {
controller: self.controller.clone(),
observers: self.observers.values().cloned().collect(),
lease_ttl_ms: self.lease_ttl_ms,
}
}
fn reconcile(&mut self, now_ms: u64) {
if self
.controller
.as_ref()
.is_some_and(|lease| lease.expires_at_ms <= now_ms)
{
self.expired_controller = self
.controller
.take()
.map(|controller| controller.client_id);
}
}
}
pub struct CoordinatedRuntime {
runtime: Arc<dyn SdkRuntime>,
leases: Mutex<RuntimeLeaseCoordinator>,
}
impl CoordinatedRuntime {
pub fn new(runtime: Arc<dyn SdkRuntime>) -> Arc<Self> {
Self::with_lease_ttl(runtime, DEFAULT_RUNTIME_LEASE_TTL_MS)
}
pub fn with_lease_ttl(runtime: Arc<dyn SdkRuntime>, lease_ttl_ms: u64) -> Arc<Self> {
Arc::new(Self {
runtime,
leases: Mutex::new(RuntimeLeaseCoordinator::new(lease_ttl_ms)),
})
}
pub fn client(
self: &Arc<Self>,
client_id: RuntimeClientId,
authorization: RuntimeAuthorization,
) -> Arc<CoordinatedRuntimeClient> {
Arc::new(CoordinatedRuntimeClient {
coordinator: self.clone(),
client_id,
authorization,
})
}
fn leases(&self) -> std::sync::MutexGuard<'_, RuntimeLeaseCoordinator> {
self.leases
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
}
pub struct CoordinatedRuntimeClient {
coordinator: Arc<CoordinatedRuntime>,
client_id: RuntimeClientId,
authorization: RuntimeAuthorization,
}
impl CoordinatedRuntimeClient {
pub fn client_id(&self) -> &RuntimeClientId {
&self.client_id
}
pub fn observe(&self) -> Result<RuntimeLeaseSnapshot, SdkError> {
self.coordinator
.leases()
.attach(self.client_id.clone(), &self.authorization, epoch_ms())
.map_err(|error| lease_sdk_error(error, SdkOperation::Events))
}
pub fn take_control(&self) -> Result<RuntimeLeaseSnapshot, SdkError> {
self.coordinator
.leases()
.take_control(self.client_id.clone(), &self.authorization, epoch_ms())
.map_err(|error| lease_sdk_error(error, SdkOperation::Input))
}
pub fn heartbeat(&self) -> Result<RuntimeLeaseSnapshot, SdkError> {
let now_ms = epoch_ms();
let mut leases = self.coordinator.leases();
let mut snapshot = leases
.heartbeat(&self.client_id, &self.authorization, now_ms)
.map_err(|error| lease_sdk_error(error, SdkOperation::Events))?;
if snapshot
.controller
.as_ref()
.is_some_and(|lease| lease.client_id == self.client_id)
{
snapshot = leases
.authorize_controller(
&self.client_id,
&self.authorization,
RuntimePermission::Interact,
now_ms,
)
.map_err(|error| lease_sdk_error(error, SdkOperation::Input))?;
}
Ok(snapshot)
}
pub fn detach(&self) -> RuntimeLeaseSnapshot {
self.coordinator
.leases()
.detach(&self.client_id, epoch_ms())
}
pub fn lease_snapshot(&self) -> Result<RuntimeLeaseSnapshot, SdkError> {
self.coordinator
.leases()
.authorize(&self.authorization, RuntimePermission::Observe, epoch_ms())
.map_err(|error| lease_sdk_error(error, SdkOperation::Events))
}
fn authorize_controller(
&self,
permission: RuntimePermission,
operation: SdkOperation,
) -> Result<(), SdkError> {
let now_ms = epoch_ms();
let mut leases = self.coordinator.leases();
leases
.claim_control(self.client_id.clone(), &self.authorization, now_ms)
.map_err(|error| lease_sdk_error(error, operation))?;
if permission != RuntimePermission::Interact {
leases
.authorize_controller(&self.client_id, &self.authorization, permission, now_ms)
.map_err(|error| lease_sdk_error(error, operation))?;
}
Ok(())
}
fn authorize_lifecycle(
&self,
permission: RuntimePermission,
operation: SdkOperation,
) -> Result<(), SdkError> {
self.coordinator
.leases()
.authorize(&self.authorization, permission, epoch_ms())
.map(|_| ())
.map_err(|error| lease_sdk_error(error, operation))
}
async fn descriptor(&self) -> Result<FrontendRuntimeDescriptor, SdkError> {
let mut descriptor = self.coordinator.runtime.describe().await?;
descriptor.actions.submit &= self.authorization.allows(RuntimePermission::Interact);
descriptor.actions.interrupt &= self.authorization.allows(RuntimePermission::Interact);
descriptor.actions.steer &= self.authorization.allows(RuntimePermission::Interact);
descriptor.actions.respond &= self.authorization.allows(RuntimePermission::Approve)
&& self.authorization.allows(RuntimePermission::Interact);
descriptor.actions.close &= self.authorization.allows(RuntimePermission::Terminate);
descriptor.actions.detach &= self.authorization.allows(RuntimePermission::Observe);
Ok(descriptor)
}
}
#[async_trait]
impl SdkRuntime for CoordinatedRuntimeClient {
async fn describe(&self) -> Result<FrontendRuntimeDescriptor, SdkError> {
self.authorize_lifecycle(RuntimePermission::Observe, SdkOperation::Events)?;
self.descriptor().await
}
async fn attach(&self, history_limit: usize) -> Result<FrontendAttachment, SdkError> {
self.observe()?;
let mut attachment = self.coordinator.runtime.attach(history_limit).await?;
attachment.descriptor = self.descriptor().await?;
Ok(attachment)
}
async fn send_input(self: Arc<Self>, prompt: String) -> Result<(), SdkError> {
self.authorize_controller(RuntimePermission::Interact, SdkOperation::Input)?;
self.coordinator.runtime.clone().send_input(prompt).await
}
async fn send_input_with_images(
self: Arc<Self>,
prompt: String,
image_urls: Vec<String>,
) -> Result<(), SdkError> {
self.authorize_controller(RuntimePermission::Interact, SdkOperation::Input)?;
self.coordinator
.runtime
.clone()
.send_input_with_images(prompt, image_urls)
.await
}
async fn submit(&self, prompt: String) -> Result<String, SdkError> {
self.authorize_controller(RuntimePermission::Interact, SdkOperation::Input)?;
self.coordinator.runtime.submit(prompt).await
}
async fn submit_with_images(
&self,
prompt: String,
image_urls: Vec<String>,
) -> Result<String, SdkError> {
self.authorize_controller(RuntimePermission::Interact, SdkOperation::Input)?;
self.coordinator
.runtime
.submit_with_images(prompt, image_urls)
.await
}
async fn interrupt(&self) -> Result<bool, SdkError> {
self.authorize_controller(RuntimePermission::Interact, SdkOperation::Interrupt)?;
self.coordinator.runtime.interrupt().await
}
async fn steer(&self, prompt: String) -> Result<(), SdkError> {
self.authorize_controller(RuntimePermission::Interact, SdkOperation::Steer)?;
self.coordinator.runtime.steer(prompt).await
}
async fn respond(&self, response: FrontendResponse) -> Result<(), SdkError> {
self.authorize_controller(RuntimePermission::Approve, SdkOperation::Respond)?;
self.coordinator.runtime.respond(response).await
}
async fn invoke(
&self,
operation: FrontendOperationInvocation,
) -> Result<FrontendOperationResult, SdkError> {
self.authorize_controller(RuntimePermission::Interact, SdkOperation::Input)?;
self.coordinator.runtime.invoke(operation).await
}
async fn lease_snapshot(&self) -> Result<RuntimeLeaseSnapshot, SdkError> {
CoordinatedRuntimeClient::lease_snapshot(self)
}
async fn take_control(&self) -> Result<RuntimeLeaseSnapshot, SdkError> {
CoordinatedRuntimeClient::take_control(self)
}
async fn heartbeat(&self) -> Result<RuntimeLeaseSnapshot, SdkError> {
CoordinatedRuntimeClient::heartbeat(self)
}
async fn detach(&self) -> Result<RuntimeLeaseSnapshot, SdkError> {
Ok(CoordinatedRuntimeClient::detach(self))
}
async fn close(&self) -> Result<(), SdkError> {
self.authorize_lifecycle(RuntimePermission::Terminate, SdkOperation::Close)?;
self.coordinator.runtime.close().await
}
}
fn lease_sdk_error(error: RuntimeLeaseError, operation: SdkOperation) -> SdkError {
match error {
error @ (RuntimeLeaseError::InvalidClientId | RuntimeLeaseError::InvalidAuthorization) => {
SdkError::InvalidArgument {
operation,
message: error.to_string(),
}
}
RuntimeLeaseError::Unauthorized(permission) => SdkError::Unauthorized {
permission: permission.as_str().into(),
},
RuntimeLeaseError::ControllerHeld {
holder,
expires_at_ms,
} => SdkError::ControllerRequired {
holder: Some(holder.to_string()),
expires_at_ms: Some(expires_at_ms),
},
RuntimeLeaseError::ControllerRequired => SdkError::ControllerRequired {
holder: None,
expires_at_ms: None,
},
RuntimeLeaseError::LeaseExpired => SdkError::LeaseExpired,
}
}
fn epoch_ms() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis()
.min(u64::MAX as u128) as u64
}
fn require(
authorization: &RuntimeAuthorization,
permission: RuntimePermission,
) -> Result<(), RuntimeLeaseError> {
if authorization.allows(permission) {
Ok(())
} else {
Err(RuntimeLeaseError::Unauthorized(permission))
}
}
#[cfg(test)]
mod tests {
use super::*;
fn client(value: &str) -> RuntimeClientId {
RuntimeClientId::parse(value).unwrap()
}
#[test]
fn many_observers_share_one_explicit_controller() {
let mut leases = RuntimeLeaseCoordinator::new(100);
let owner = RuntimeAuthorization::owner();
let observer = RuntimeAuthorization::observer();
leases.attach(client("viewer-a"), &observer, 10).unwrap();
leases.attach(client("viewer-b"), &observer, 11).unwrap();
let snapshot = leases.claim_control(client("owner"), &owner, 12).unwrap();
assert_eq!(snapshot.observers.len(), 3);
assert_eq!(snapshot.controller.unwrap().client_id, client("owner"));
let error = leases
.claim_control(client("viewer-b"), &owner, 13)
.unwrap_err();
assert!(matches!(error, RuntimeLeaseError::ControllerHeld { .. }));
}
#[test]
fn observer_cannot_claim_control_approve_or_terminate() {
let mut leases = RuntimeLeaseCoordinator::new(100);
let observer = RuntimeAuthorization::observer();
leases.attach(client("viewer"), &observer, 1).unwrap();
assert_eq!(
leases
.claim_control(client("viewer"), &observer, 2)
.unwrap_err(),
RuntimeLeaseError::Unauthorized(RuntimePermission::Interact)
);
assert_eq!(
leases
.authorize_controller(&client("viewer"), &observer, RuntimePermission::Approve, 2)
.unwrap_err(),
RuntimeLeaseError::Unauthorized(RuntimePermission::Approve)
);
assert_eq!(
leases
.authorize(&observer, RuntimePermission::Terminate, 2)
.unwrap_err(),
RuntimeLeaseError::Unauthorized(RuntimePermission::Terminate)
);
}
#[test]
fn expiry_is_deterministic_and_requires_a_new_claim() {
let mut leases = RuntimeLeaseCoordinator::new(10);
let owner = RuntimeAuthorization::owner();
leases.claim_control(client("a"), &owner, 5).unwrap();
assert_eq!(
leases.claim_control(client("a"), &owner, 15).unwrap_err(),
RuntimeLeaseError::LeaseExpired
);
assert_eq!(
leases
.authorize_controller(&client("a"), &owner, RuntimePermission::Interact, 15)
.unwrap_err(),
RuntimeLeaseError::LeaseExpired
);
let snapshot = leases.claim_control(client("b"), &owner, 15).unwrap();
assert_eq!(snapshot.controller.unwrap().client_id, client("b"));
}
#[test]
fn successful_mutation_renews_controller_and_observer_activity() {
let mut leases = RuntimeLeaseCoordinator::new(10);
let owner = RuntimeAuthorization::owner();
leases.claim_control(client("a"), &owner, 5).unwrap();
let snapshot = leases
.authorize_controller(&client("a"), &owner, RuntimePermission::Approve, 9)
.unwrap();
assert_eq!(snapshot.controller.unwrap().expires_at_ms, 19);
assert_eq!(snapshot.observers[0].last_seen_ms, 9);
}
#[test]
fn takeover_and_detach_are_explicit_and_release_control() {
let mut leases = RuntimeLeaseCoordinator::new(10);
let owner = RuntimeAuthorization::owner();
leases.claim_control(client("a"), &owner, 1).unwrap();
let snapshot = leases.take_control(client("b"), &owner, 2).unwrap();
assert_eq!(snapshot.controller.unwrap().client_id, client("b"));
let snapshot = leases.detach(&client("b"), 3);
assert!(snapshot.controller.is_none());
assert_eq!(
snapshot
.observers
.into_iter()
.map(|observer| observer.client_id)
.collect::<Vec<_>>(),
vec![client("a")]
);
}
#[test]
fn client_ids_are_opaque_bounded_and_header_safe() {
for invalid in ["", "space here", "slash/here", "💥"] {
assert_eq!(
RuntimeClientId::parse(invalid).unwrap_err(),
RuntimeLeaseError::InvalidClientId
);
}
assert_eq!(
RuntimeClientId::parse("a".repeat(129)).unwrap_err(),
RuntimeLeaseError::InvalidClientId
);
assert_eq!(
RuntimeClientId::parse("client-1.v2_ok").unwrap().as_str(),
"client-1.v2_ok"
);
let owner = RuntimeAuthorization::owner();
let requested = RuntimeAuthorization::parse_header("observe,interact").unwrap();
assert_eq!(
owner.restrict_to(&requested).header_value(),
"observe,interact"
);
assert_eq!(
RuntimeAuthorization::parse_header("observe,admin").unwrap_err(),
RuntimeLeaseError::InvalidAuthorization
);
}
}