use std::borrow::Cow;
use std::collections::VecDeque;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use futures::{SinkExt, StreamExt};
use micromux::{
ChangeKind, CommandRejection, PtyInputSendError, SchedulerStopped, ServiceCommandResult,
ServiceSnapshot, SessionChange, SessionModelReader, TerminalSize,
};
use serde::Serialize;
use tokio::io::{AsyncRead, AsyncWrite};
use tokio::sync::Semaphore;
use tokio_util::sync::CancellationToken;
#[path = "unix/log_reads.rs"]
mod log_reads;
use super::ControlServer;
use crate::endpoint::{CanonicalConfigPath, ControlEndpoint, project_lock_path};
use crate::protocol::{
ErrorCode, PROTOCOL_VERSION, Request, Response, ServiceBrief, SessionInfo,
supports_versioned_subscriptions,
};
use crate::{
ControlError, Framing, IDLE_TIMEOUT, REQUEST_TIMEOUT, SUBSCRIPTION_HEARTBEAT_INTERVAL, framed,
read_message, write_message,
};
const DEFAULT_LOG_TAIL: usize = 200;
const MAX_LOG_TAIL: usize = 2000;
const DEFAULT_EVENT_TAIL: usize = 50;
const MAX_EVENT_TAIL: usize = micromux::EVENT_HISTORY;
const LAG_REPLAY_SERVICE_KINDS: [ChangeKind; 4] = [
ChangeKind::Status,
ChangeKind::Logs,
ChangeKind::Health,
ChangeKind::Events,
];
const RESPONSE_MAX_BYTES: usize = 512 * 1024;
const MAX_CONNECTIONS: usize = 64;
const MAX_UNSOLICITED_SUBSCRIPTION_FRAMES_PER_WINDOW: usize = 32;
const UNSOLICITED_SUBSCRIPTION_FRAME_WINDOW: std::time::Duration =
std::time::Duration::from_mins(1);
const SUBSCRIPTION_WRITE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2);
const OWNER_PROBE_LOCK_WAIT_LIMIT: std::time::Duration = std::time::Duration::from_secs(5);
const OWNER_PROBE_LOCK_RETRY: std::time::Duration = std::time::Duration::from_millis(1);
#[derive(Clone, Copy)]
struct SubscriptionPolicy {
keepalive_interval: std::time::Duration,
keepalive_kind: ChangeKind,
inbound_frame_window: std::time::Duration,
}
impl SubscriptionPolicy {
fn legacy() -> Self {
Self {
keepalive_interval: IDLE_TIMEOUT,
keepalive_kind: ChangeKind::Events,
inbound_frame_window: UNSOLICITED_SUBSCRIPTION_FRAME_WINDOW,
}
}
fn versioned() -> Self {
Self {
keepalive_interval: SUBSCRIPTION_HEARTBEAT_INTERVAL,
keepalive_kind: ChangeKind::Heartbeat,
inbound_frame_window: UNSOLICITED_SUBSCRIPTION_FRAME_WINDOW,
}
}
}
pub struct EndpointGuard {
listener: tokio::net::UnixListener,
socket_path: PathBuf,
_lock: std::fs::File,
_project_lock: Option<std::fs::File>,
}
impl Drop for EndpointGuard {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.socket_path);
}
}
struct InboundFrameRate {
accepted_at: VecDeque<tokio::time::Instant>,
}
impl InboundFrameRate {
fn new() -> Self {
Self {
accepted_at: VecDeque::with_capacity(MAX_UNSOLICITED_SUBSCRIPTION_FRAMES_PER_WINDOW),
}
}
fn accepts(&mut self, now: tokio::time::Instant, window: std::time::Duration) -> bool {
while self
.accepted_at
.front()
.is_some_and(|accepted| now.saturating_duration_since(*accepted) >= window)
{
self.accepted_at.pop_front();
}
if self.accepted_at.len() >= MAX_UNSOLICITED_SUBSCRIPTION_FRAMES_PER_WINDOW {
return false;
}
self.accepted_at.push_back(now);
true
}
}
pub(super) fn bind(endpoint: &ControlEndpoint) -> Result<Option<EndpointGuard>, ControlError> {
match endpoint {
ControlEndpoint::Unix(path) => bind_unix(path, None),
ControlEndpoint::WindowsNamedPipe(_) => Err(ControlError::Unsupported),
}
}
pub(super) fn claim_project(
config_path: &CanonicalConfigPath,
) -> Result<super::ProjectClaim, super::ClaimError> {
claim_project_at(&project_lock_path(config_path).map_err(ControlError::from)?)
}
pub(super) fn claim_project_at(lock_path: &Path) -> Result<super::ProjectClaim, super::ClaimError> {
use std::io::{Read as _, Seek as _, Write as _};
use std::os::unix::fs::{OpenOptionsExt as _, PermissionsExt as _};
let mut project_lock = std::fs::OpenOptions::new()
.create(true)
.read(true)
.write(true)
.truncate(false)
.mode(0o600)
.open(lock_path)
.map_err(ControlError::from)?;
project_lock
.set_permissions(std::fs::Permissions::from_mode(0o600))
.map_err(ControlError::from)?;
match fs2::FileExt::try_lock_exclusive(&project_lock) {
Ok(()) => {}
Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {
let mut recorded = String::new();
let pid = project_lock
.read_to_string(&mut recorded)
.ok()
.and_then(|_| recorded.trim().parse().ok());
return Err(super::ClaimError::Held { pid });
}
Err(err) => return Err(ControlError::from(err).into()),
}
if let Err(err) = project_lock
.set_len(0)
.and_then(|()| project_lock.rewind())
.and_then(|()| write!(project_lock, "{}", std::process::id()))
{
tracing::debug!(%err, "cannot record this session's pid in its owner lock");
}
Ok(super::ProjectClaim {
lock: Some(project_lock),
})
}
pub(super) fn bind_project(
endpoint: &ControlEndpoint,
claim: &super::ProjectClaim,
) -> Result<Option<EndpointGuard>, ControlError> {
let project_lock = claim
.lock
.as_ref()
.map(std::fs::File::try_clone)
.transpose()?;
match endpoint {
ControlEndpoint::Unix(path) => bind_unix(path, project_lock),
ControlEndpoint::WindowsNamedPipe(_) => Err(ControlError::Unsupported),
}
}
pub(super) fn endpoint_owner_lock_held(endpoint: &ControlEndpoint) -> Result<bool, ControlError> {
match endpoint {
ControlEndpoint::Unix(path) => owner_lock_held_unix(path),
ControlEndpoint::WindowsNamedPipe(_) => Err(ControlError::Unsupported),
}
}
fn open_endpoint_lock(socket_path: &Path) -> std::io::Result<std::fs::File> {
use std::os::unix::fs::{OpenOptionsExt as _, PermissionsExt as _};
let lock_file = std::fs::OpenOptions::new()
.create(true)
.read(true)
.write(true)
.truncate(false)
.mode(0o600)
.open(socket_path.with_extension("lock"))?;
lock_file.set_permissions(std::fs::Permissions::from_mode(0o600))?;
Ok(lock_file)
}
fn bind_unix(
socket_path: &Path,
project_lock: Option<std::fs::File>,
) -> Result<Option<EndpointGuard>, ControlError> {
use std::os::unix::fs::PermissionsExt as _;
let lock_file = open_endpoint_lock(socket_path)?;
if !try_lock_endpoint_owner(&lock_file)? {
return Ok(None);
}
if let Err(err) = std::fs::remove_file(socket_path)
&& err.kind() != std::io::ErrorKind::NotFound
{
return Err(err.into());
}
let listener = tokio::net::UnixListener::bind(socket_path)?;
std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o600))?;
Ok(Some(EndpointGuard {
listener,
socket_path: socket_path.to_path_buf(),
_lock: lock_file,
_project_lock: project_lock,
}))
}
fn try_lock_endpoint_owner(lock_file: &std::fs::File) -> std::io::Result<bool> {
let would_block = |err: &std::io::Error| err.kind() == std::io::ErrorKind::WouldBlock;
let deadline = std::time::Instant::now() + OWNER_PROBE_LOCK_WAIT_LIMIT;
loop {
match fs2::FileExt::try_lock_exclusive(lock_file) {
Ok(()) => return Ok(true),
Err(err) if would_block(&err) => {}
Err(err) => return Err(err),
}
match fs2::FileExt::try_lock_shared(lock_file) {
Ok(()) => fs2::FileExt::unlock(lock_file)?,
Err(err) if would_block(&err) => return Ok(false),
Err(err) => return Err(err),
}
if std::time::Instant::now() >= deadline {
return Ok(false);
}
std::thread::sleep(OWNER_PROBE_LOCK_RETRY);
}
}
pub(super) fn prune_dead_socket(socket_path: &Path) -> std::io::Result<bool> {
let lock_file = open_endpoint_lock(socket_path)?;
match fs2::FileExt::try_lock_shared(&lock_file) {
Ok(()) => {}
Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => return Ok(false),
Err(err) => return Err(err),
}
let removed = match std::fs::remove_file(socket_path) {
Ok(()) => Ok(true),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(false),
Err(err) => Err(err),
};
drop(lock_file);
removed
}
fn owner_lock_held_unix(socket_path: &Path) -> Result<bool, ControlError> {
let lock_path = socket_path.with_extension("lock");
let lock_file = match std::fs::OpenOptions::new()
.read(true)
.write(true)
.open(&lock_path)
{
Ok(lock_file) => lock_file,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(false),
Err(err) => return Err(err.into()),
};
match fs2::FileExt::try_lock_shared(&lock_file) {
Ok(()) => Ok(false),
Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => Ok(true),
Err(err) => Err(err.into()),
}
}
pub(super) async fn serve(
server: Arc<ControlServer>,
guard: EndpointGuard,
shutdown: CancellationToken,
) -> Result<(), ControlError> {
let connections = Arc::new(Semaphore::new(MAX_CONNECTIONS));
loop {
tokio::select! {
() = shutdown.cancelled() => break,
accepted = guard.listener.accept() => {
match accepted {
Ok((stream, _addr)) => {
let Ok(permit) = Arc::clone(&connections).try_acquire_owned() else {
tracing::debug!("control: connection limit reached");
continue;
};
let server = Arc::clone(&server);
let shutdown = shutdown.clone();
tokio::spawn(async move {
let _permit = permit;
handle_connection(server, stream, shutdown).await;
});
}
Err(err) => {
tracing::warn!(?err, "control: accept failed");
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
}
}
}
}
}
drop(guard);
Ok(())
}
async fn handle_connection<S>(server: Arc<ControlServer>, stream: S, shutdown: CancellationToken)
where
S: AsyncRead + AsyncWrite + Unpin + Send,
{
let mut conn = framed(stream);
loop {
let read = tokio::select! {
() = shutdown.cancelled() => return,
read = tokio::time::timeout(IDLE_TIMEOUT, read_message::<S, Request>(&mut conn)) => read,
};
let request = match read {
Ok(Ok(Some(request))) => request,
Ok(Err(err)) => {
tracing::debug!(?err, "control: rejecting bad request frame");
let response = Response::error(ErrorCode::BadRequest, err.to_string());
let _ = write_response(&mut conn, &response, &shutdown).await;
return;
}
Ok(Ok(None)) | Err(_) => return,
};
match request {
Request::Subscribe => {
stream_changes(&server, conn, shutdown, SubscriptionPolicy::legacy()).await;
return;
}
Request::SubscribeWithVersion { protocol_version } => {
let policy = if supports_versioned_subscriptions(protocol_version) {
SubscriptionPolicy::versioned()
} else {
SubscriptionPolicy::legacy()
};
stream_changes(&server, conn, shutdown, policy).await;
return;
}
_ => {}
}
if matches!(request, Request::Shutdown) {
let _ = write_response(&mut conn, &Response::ShuttingDown, &shutdown).await;
shutdown.cancel();
return;
}
let response = tokio::select! {
() = shutdown.cancelled() => return,
dispatched = tokio::time::timeout(REQUEST_TIMEOUT, dispatch(&server, request)) => {
match dispatched {
Ok(response) => response,
Err(_) => {
Response::error(ErrorCode::Timeout, "the scheduler did not respond in time")
}
}
}
};
if !write_response(&mut conn, &response, &shutdown).await {
return;
}
}
}
async fn write_response<S>(
conn: &mut Framing<S>,
response: &Response,
shutdown: &CancellationToken,
) -> bool
where
S: AsyncRead + AsyncWrite + Unpin,
{
let response = response_within_frame(response);
tokio::select! {
() = shutdown.cancelled() => false,
result = tokio::time::timeout(REQUEST_TIMEOUT, write_message(conn, response.as_ref())) => {
matches!(result, Ok(Ok(())))
}
}
}
fn response_within_frame(response: &Response) -> Cow<'_, Response> {
if encoded_len(response).is_some_and(|len| len <= RESPONSE_MAX_BYTES) {
return Cow::Borrowed(response);
}
if let Response::Description(info) = response
&& let Some(info) = bounded_description(info)
{
return Cow::Owned(Response::Description(info));
}
Cow::Owned(Response::error(
ErrorCode::LimitExceeded,
"the response exceeds the control protocol frame budget",
))
}
fn bounded_description(info: &SessionInfo) -> Option<SessionInfo> {
let mut info = info.clone();
let services = std::mem::take(&mut info.services);
info.services_truncated = true;
let mut low = 0;
let mut high = services.len();
while low < high {
let middle = low + (high - low).div_ceil(2);
info.services = services.get(..middle)?.to_vec();
if encoded_len(&Response::Description(info.clone()))
.is_some_and(|len| len <= RESPONSE_MAX_BYTES)
{
low = middle;
} else {
high = middle - 1;
}
}
info.services = services.get(..low)?.to_vec();
encoded_len(&Response::Description(info.clone()))
.is_some_and(|len| len <= RESPONSE_MAX_BYTES)
.then_some(info)
}
async fn stream_changes<S>(
server: &ControlServer,
conn: Framing<S>,
shutdown: CancellationToken,
policy: SubscriptionPolicy,
) where
S: AsyncRead + AsyncWrite + Unpin + Send,
{
use tokio::sync::broadcast::error::RecvError;
let (mut sink, mut stream) = conn.split();
let mut changes = server.reader.subscribe();
let keepalive = tokio::time::sleep(policy.keepalive_interval);
tokio::pin!(keepalive);
let mut inbound_rate = InboundFrameRate::new();
'stream: loop {
tokio::select! {
() = shutdown.cancelled() => break,
() = &mut keepalive => {
let change = SessionChange {
service_id: SessionChange::SESSION_WIDE.to_string(),
kind: policy.keepalive_kind,
};
if !send_stream_change(&mut sink, change, &shutdown).await {
break;
}
keepalive
.as_mut()
.reset(tokio::time::Instant::now() + policy.keepalive_interval);
}
incoming = stream.next() => {
match incoming {
None | Some(Err(_)) => break,
Some(Ok(_)) => {
let now = tokio::time::Instant::now();
if !inbound_rate.accepts(now, policy.inbound_frame_window) {
break;
}
}
}
}
change = changes.recv() => {
match change {
Ok(change) => {
if !send_stream_change(&mut sink, change, &shutdown).await {
break;
}
keepalive
.as_mut()
.reset(tokio::time::Instant::now() + policy.keepalive_interval);
}
Err(RecvError::Lagged(_)) => {
let snapshots = server.reader.services();
for change in lag_replay_changes(&snapshots) {
if !send_stream_change(&mut sink, change, &shutdown).await {
break 'stream;
}
}
keepalive
.as_mut()
.reset(tokio::time::Instant::now() + policy.keepalive_interval);
}
Err(RecvError::Closed) => break,
}
}
}
}
}
async fn send_stream_change<S>(
sink: &mut futures::stream::SplitSink<Framing<S>, String>,
change: SessionChange,
shutdown: &CancellationToken,
) -> bool
where
S: AsyncRead + AsyncWrite + Unpin,
{
let Ok(line) = serde_json::to_string(&Response::Change(change)) else {
return false;
};
tokio::select! {
() = shutdown.cancelled() => false,
result = tokio::time::timeout(SUBSCRIPTION_WRITE_TIMEOUT, sink.send(line)) => {
matches!(result, Ok(Ok(())))
}
}
}
fn lag_replay_changes(snapshots: &[ServiceSnapshot]) -> impl Iterator<Item = SessionChange> + '_ {
std::iter::once(SessionChange {
service_id: SessionChange::SESSION_WIDE.to_string(),
kind: ChangeKind::Roster,
})
.chain(snapshots.iter().flat_map(|snapshot| {
LAG_REPLAY_SERVICE_KINDS.map(|kind| SessionChange {
service_id: snapshot.id.clone(),
kind,
})
}))
}
async fn dispatch(server: &ControlServer, request: Request) -> Response {
match request {
Request::Describe => Response::Description(describe(server)),
Request::ListServices => Response::Services(server.reader.services()),
Request::GetService { service } => get_service(&server.reader, &service),
Request::GetLogs {
service,
run_generation,
tail,
} => {
if run_generation.is_none() {
get_logs(&server.reader, &service, None, tail)
} else {
let reader = server.reader.clone();
log_reads::run(move || get_logs(&reader, &service, run_generation, tail)).await
}
}
Request::FollowLogs {
service,
run_generation,
after,
} => {
if run_generation.is_none() {
follow_logs(&server.reader, &service, None, after)
} else {
let reader = server.reader.clone();
log_reads::run(move || follow_logs(&reader, &service, run_generation, after)).await
}
}
Request::ListLogRuns { service } => {
let reader = server.reader.clone();
log_reads::run(move || list_log_runs(&reader, &service)).await
}
Request::GetHealth { service } => {
if server.reader.service(&service).is_none() {
return unknown_service(&service);
}
let mut attempt = server.reader.latest_health(&service);
bound_health_attempt(&mut attempt);
Response::Health(attempt)
}
Request::GetHealthHistory { service } => get_health_history(&server.reader, &service),
Request::GetEvents {
service,
after,
tail,
} => get_events(&server.reader, &service, after, tail),
Request::Restart { service } => acknowledge(server.control.restart(&service).await),
Request::RestartAll => acknowledge(server.control.restart_all().await),
Request::Enable { service } => acknowledge(server.control.enable(&service).await),
Request::Disable { service } => acknowledge(server.control.disable(&service).await),
Request::StartDynamicService { params } => {
acknowledge_dynamic(server.control.start_dynamic(params).await)
}
Request::ReplaceDynamicService {
service,
expected_revision,
params,
} => acknowledge_dynamic(
server
.control
.replace_dynamic(&service, expected_revision, params)
.await,
),
Request::RenewDynamicService {
service,
expected_revision,
expires_after,
} => acknowledge_dynamic(
server
.control
.renew_dynamic(&service, expected_revision, expires_after)
.await,
),
Request::StopDynamicService { service } => {
acknowledge_dynamic(server.control.stop_dynamic(&service).await)
}
Request::GetEnvironment { service } => environment(server, service.as_ref()).await,
Request::ReconcileConfig { dry_run } => reconcile(server, dry_run).await,
Request::SendInput {
service,
data,
paste,
} => send_input(server, service, data.into_inner(), paste).await,
Request::ResizeTerminal { cols, rows } => resize_terminal(server, cols, rows).await,
Request::Subscribe | Request::SubscribeWithVersion { .. } => Response::error(
ErrorCode::BadRequest,
"subscribe must be the only request on a connection",
),
Request::Shutdown => {
Response::error(ErrorCode::BadRequest, "shutdown is handled before dispatch")
}
}
}
async fn send_input(
server: &ControlServer,
service: micromux::ServiceID,
data: Vec<u8>,
paste: bool,
) -> Response {
let Some(terminal) = &server.terminal else {
return no_terminal();
};
let Some(snapshot) = server.reader.service(&service) else {
return unknown_service(&service);
};
if !matches!(
snapshot.execution,
micromux::Execution::Running | micromux::Execution::Stopping
) {
let kind = if paste { "paste" } else { "input" };
return Response::error(
ErrorCode::InvalidState,
format!("terminal {kind} discarded: {service} is not running"),
);
}
let prepared = if paste {
terminal.prepare_paste(service, data)
} else {
terminal.prepare_input(service, data)
};
let prepared = match prepared {
Ok(prepared) => prepared,
Err(err) => return Response::error(ErrorCode::LimitExceeded, err.to_string()),
};
match terminal.send(prepared).await {
Ok(()) => Response::TerminalQueued,
Err(
PtyInputSendError::Closed(_)
| PtyInputSendError::Full(_)
| PtyInputSendError::Disconnected(_),
) => Response::error(ErrorCode::SchedulerStopped, "the scheduler has stopped"),
Err(PtyInputSendError::WrongSession(_)) => Response::error(
ErrorCode::Internal,
"terminal input was prepared for another session",
),
}
}
async fn resize_terminal(server: &ControlServer, cols: u16, rows: u16) -> Response {
let Some(terminal) = &server.terminal else {
return no_terminal();
};
if cols == 0 || rows == 0 {
return Response::error(
ErrorCode::BadRequest,
"a terminal needs at least one column and one row",
);
}
if terminal.resize(TerminalSize { cols, rows }).await {
Response::TerminalQueued
} else {
Response::error(ErrorCode::SchedulerStopped, "the scheduler has stopped")
}
}
fn no_terminal() -> Response {
Response::error(
ErrorCode::PolicyDenied,
"this session does not accept terminal input",
)
}
async fn reconcile(server: &ControlServer, dry_run: bool) -> Response {
acknowledge_reconcile(server.control.reconcile_config(dry_run).await)
}
async fn environment(server: &ControlServer, service: Option<µmux::ServiceID>) -> Response {
acknowledge_with(
server.control.environment(service).await,
Response::Environment,
)
}
fn get_service(reader: &SessionModelReader, service: &str) -> Response {
let snapshots = reader.services();
snapshots
.iter()
.find(|snapshot| snapshot.id == service)
.or_else(|| snapshots.iter().find(|snapshot| snapshot.name == service))
.cloned()
.map_or_else(
|| unknown_service(service),
|snapshot| Response::Service(Box::new(snapshot)),
)
}
fn get_events(
reader: &SessionModelReader,
service: &str,
after: Option<u64>,
tail: Option<usize>,
) -> Response {
if reader.service(service).is_none() {
return unknown_service(service);
}
let requested_tail = tail.unwrap_or(DEFAULT_EVENT_TAIL);
let (mut events, mut truncated) =
reader.events(service, after, Some(requested_tail.min(MAX_EVENT_TAIL)));
bound_events(&mut events, &mut truncated, after.is_none());
Response::Events { events, truncated }
}
fn list_log_runs(reader: &SessionModelReader, service: &str) -> Response {
if reader.service(service).is_none() {
return unknown_service(service);
}
Response::LogRuns {
runs: reader.log_runs(service),
}
}
fn get_logs(
reader: &SessionModelReader,
service: &str,
run_generation: Option<u64>,
tail: Option<usize>,
) -> Response {
if reader.service(service).is_none() {
return unknown_service(service);
}
let requested_tail = tail.unwrap_or(if run_generation.is_some() {
MAX_LOG_TAIL
} else {
DEFAULT_LOG_TAIL
});
let tail = requested_tail.min(MAX_LOG_TAIL);
let mut truncated = false;
let mut lines = match run_generation {
Some(run_generation) => {
let run = match reader.try_run_log(service, run_generation, Some(tail)) {
Ok(Some(run)) => run,
Ok(None) => return unknown_run(service, run_generation),
Err(err) => return log_read_error(err),
};
run.lines
}
None => reader.logs(service, Some(tail)),
};
truncated |= requested_tail > MAX_LOG_TAIL && lines.len() >= tail;
if let Some(run_generation) = run_generation
&& tail == MAX_LOG_TAIL
{
truncated |= reader
.log_runs(service)
.into_iter()
.find(|run| run.run_generation == run_generation)
.is_some_and(|run| run.line_count > lines.len());
}
let first_retained_seq = run_generation
.is_none()
.then(|| reader.first_retained_log_seq(service))
.flatten();
truncated |= bound_tail_response_lines(&mut lines, first_retained_seq);
logs_response(lines, truncated, first_retained_seq)
}
fn follow_logs(
reader: &SessionModelReader,
service: &str,
run_generation: Option<u64>,
after: Option<u64>,
) -> Response {
if reader.service(service).is_none() {
return unknown_service(service);
}
if let Some(run_generation) = run_generation {
let run = match reader.try_run_log_after(
service,
run_generation,
after,
Some(MAX_LOG_TAIL + 1),
) {
Ok(Some(run)) => run,
Ok(None) => return unknown_run(service, run_generation),
Err(err) => return log_read_error(err),
};
let mut lines = run.lines;
if let Some(cursor) = after {
lines.retain(|line| line.seq > cursor);
}
let (lines, truncated) = bound_follow_response_lines_page(lines, None);
return logs_response(lines, truncated, None);
}
if let Some(cursor) = after {
let (first_retained_seq, lines) = reader.logs_since(service, cursor);
let (lines, truncated) = bound_follow_response_lines_page(lines, first_retained_seq);
return logs_response(lines, truncated, first_retained_seq);
}
let mut lines = reader.logs(service, None);
let first_retained_seq = lines.first().map(|line| line.seq);
let mut truncated = false;
if lines.len() > MAX_LOG_TAIL {
let drop = lines.len() - MAX_LOG_TAIL;
lines.drain(0..drop);
truncated = true;
}
truncated |= bound_tail_response_lines(&mut lines, first_retained_seq);
logs_response(lines, truncated, first_retained_seq)
}
fn bound_follow_response_lines_page(
mut lines: Vec<micromux::LogLine>,
first_retained_seq: Option<u64>,
) -> (Vec<micromux::LogLine>, bool) {
let capped = lines.len() > MAX_LOG_TAIL;
lines.truncate(MAX_LOG_TAIL);
let truncated = capped || bound_follow_response_lines(&mut lines, first_retained_seq);
(lines, truncated)
}
fn describe(server: &ControlServer) -> SessionInfo {
let snapshots = server.reader.services();
let services = snapshots
.iter()
.map(|snapshot| ServiceBrief {
id: snapshot.id.clone(),
name: snapshot.name.clone(),
})
.collect();
let dynamic_services = server.dynamic_policy.enabled.then(|| {
let live_services = snapshots
.iter()
.filter(|snapshot| {
snapshot.origin == micromux::OriginKind::Dynamic && snapshot.retired.is_none()
})
.count();
crate::DynamicServicesCaps {
max_services: server.dynamic_policy.max_services,
live_services,
max_lifetime_secs: server
.dynamic_policy
.max_lifetime
.map(|value| value.as_secs()),
allowed_working_roots: server
.dynamic_policy
.allowed_working_roots
.iter()
.map(|root| root.display().to_string())
.collect(),
}
});
SessionInfo {
protocol_version: PROTOCOL_VERSION,
id: server.identity.id.clone(),
pid: server.identity.pid,
start_time: server.identity.start_time,
name: server.identity.name.clone(),
working_dir: server.identity.working_dir.clone(),
config_path: server.identity.config_path.clone(),
services,
services_truncated: false,
micromux_version: server.identity.micromux_version.clone(),
session: server
.session
.as_ref()
.map(|status| Box::new(status.current())),
capabilities: Some(crate::SessionCapabilities {
dynamic_services,
disk_log_reads: Some(log_reads::health()),
}),
}
}
fn unknown_service(service: &str) -> Response {
Response::error(
ErrorCode::UnknownService,
format!("unknown service `{service}`"),
)
}
fn log_read_error(error: micromux::LogRunReadError) -> Response {
Response::error(ErrorCode::Busy, error.to_string())
}
fn unknown_run(service: &str, run_generation: u64) -> Response {
Response::error(
ErrorCode::UnknownRun,
format!("service `{service}` has no retained run `{run_generation}`"),
)
}
fn logs_response(
lines: Vec<micromux::LogLine>,
truncated: bool,
first_retained_seq: Option<u64>,
) -> Response {
Response::Logs {
lines,
truncated,
first_retained_seq,
}
}
fn bound_tail_response_lines(
lines: &mut Vec<micromux::LogLine>,
first_retained_seq: Option<u64>,
) -> bool {
if logs_fit(lines, true, first_retained_seq) {
return false;
}
let mut low = 0;
let mut high = lines.len();
while low < high {
let middle = low + (high - low) / 2;
if lines
.get(middle..)
.is_some_and(|lines| logs_fit(lines, true, first_retained_seq))
{
high = middle;
} else {
low = middle + 1;
}
}
if low == lines.len() {
let retained = lines
.last()
.cloned()
.and_then(|line| fit_single_log_line(line, true, first_retained_seq));
lines.clear();
lines.extend(retained);
} else {
lines.drain(..low);
}
true
}
fn bound_follow_response_lines(
lines: &mut Vec<micromux::LogLine>,
first_retained_seq: Option<u64>,
) -> bool {
if logs_fit(lines, true, first_retained_seq) {
return false;
}
let mut low = 0;
let mut high = lines.len();
while low < high {
let middle = low + (high - low).div_ceil(2);
if lines
.get(..middle)
.is_some_and(|lines| logs_fit(lines, true, first_retained_seq))
{
low = middle;
} else {
high = middle - 1;
}
}
if low == 0 {
let retained = lines
.first()
.cloned()
.and_then(|line| fit_single_log_line(line, false, first_retained_seq));
lines.clear();
lines.extend(retained);
} else {
lines.truncate(low);
}
true
}
fn fit_single_log_line(
mut line: micromux::LogLine,
keep_tail: bool,
first_retained_seq: Option<u64>,
) -> Option<micromux::LogLine> {
let original = std::mem::take(&mut line.line);
let mut boundaries = if keep_tail {
original
.char_indices()
.map(|(index, _)| original.len().saturating_sub(index))
.collect::<Vec<_>>()
} else {
original
.char_indices()
.map(|(index, _)| index)
.collect::<Vec<_>>()
};
boundaries.push(0);
boundaries.push(original.len());
boundaries.sort_unstable();
boundaries.dedup();
let mut low = 0usize;
let mut high = boundaries.len().saturating_sub(1);
while low < high {
let middle = low + (high - low).div_ceil(2);
let bytes = boundaries.get(middle).copied().unwrap_or_default();
line.line = if keep_tail {
original
.get(original.len().saturating_sub(bytes)..)
.unwrap_or_default()
.to_string()
} else {
original.get(..bytes).unwrap_or_default().to_string()
};
if logs_fit(std::slice::from_ref(&line), true, first_retained_seq) {
low = middle;
} else {
high = middle - 1;
}
}
let bytes = boundaries.get(low).copied().unwrap_or_default();
line.line = if keep_tail {
original
.get(original.len().saturating_sub(bytes)..)
.unwrap_or_default()
.to_string()
} else {
original.get(..bytes).unwrap_or_default().to_string()
};
logs_fit(std::slice::from_ref(&line), true, first_retained_seq).then_some(line)
}
fn logs_fit(lines: &[micromux::LogLine], truncated: bool, first_retained_seq: Option<u64>) -> bool {
encoded_len(&Response::Logs {
lines: lines.to_vec(),
truncated,
first_retained_seq,
})
.is_some_and(|len| len <= RESPONSE_MAX_BYTES)
}
fn encoded_len(value: &impl Serialize) -> Option<usize> {
serde_json::to_vec(value).ok().map(|encoded| encoded.len())
}
fn bound_health_attempt(attempt: &mut Option<micromux::HealthAttempt>) {
if health_response_fits(attempt.as_ref()) {
return;
}
let Some(attempt) = attempt else {
return;
};
trim_health_attempt_to_fit(attempt, |attempt| health_response_fits(Some(attempt)));
}
fn get_health_history(reader: &SessionModelReader, service: &str) -> Response {
if reader.service(service).is_none() {
return unknown_service(service);
}
let mut attempts = reader.healthchecks(service);
bound_health_history(&mut attempts);
Response::HealthHistory { attempts }
}
fn health_response_fits(attempt: Option<µmux::HealthAttempt>) -> bool {
encoded_len(&Response::Health(attempt.cloned())).is_some_and(|len| len <= RESPONSE_MAX_BYTES)
}
fn trim_health_attempt_to_fit(
attempt: &mut micromux::HealthAttempt,
fits: impl Fn(µmux::HealthAttempt) -> bool,
) {
let output = std::mem::take(&mut attempt.output);
let mut low = 0usize;
let mut high = output.len();
while low < high {
let middle = low + (high - low).div_ceil(2);
attempt.output = output
.get(output.len().saturating_sub(middle)..)
.unwrap_or_default()
.to_vec();
if fits(attempt) {
low = middle;
} else {
high = middle - 1;
}
}
attempt.output = output
.get(output.len().saturating_sub(low)..)
.unwrap_or_default()
.to_vec();
if fits(attempt) {
return;
}
let command = std::mem::take(&mut attempt.command);
let mut boundaries = command
.char_indices()
.map(|(index, _)| index)
.collect::<Vec<_>>();
boundaries.push(command.len());
let mut low = 0usize;
let mut high = boundaries.len().saturating_sub(1);
while low < high {
let middle = low + (high - low) / 2;
let start = boundaries.get(middle).copied().unwrap_or(command.len());
attempt.command = command.get(start..).unwrap_or_default().to_string();
if fits(attempt) {
high = middle;
} else {
low = middle + 1;
}
}
let start = boundaries.get(low).copied().unwrap_or(command.len());
attempt.command = command.get(start..).unwrap_or_default().to_string();
}
fn bound_health_history(attempts: &mut Vec<micromux::HealthAttempt>) {
if health_history_fits(attempts) {
return;
}
let original = std::mem::take(attempts);
let mut low = 0usize;
let mut high = original.len();
while low < high {
let middle = low + (high - low).div_ceil(2);
*attempts = original
.get(original.len().saturating_sub(middle)..)
.unwrap_or_default()
.to_vec();
if health_history_fits(attempts) {
low = middle;
} else {
high = middle - 1;
}
}
*attempts = original
.get(original.len().saturating_sub(low.max(1))..)
.unwrap_or_default()
.to_vec();
if !health_history_fits(attempts)
&& let Some(attempt) = attempts.first_mut()
{
trim_health_attempt_to_fit(attempt, |attempt| {
health_history_fits(std::slice::from_ref(attempt))
});
}
}
fn health_history_fits(attempts: &[micromux::HealthAttempt]) -> bool {
encoded_len(&Response::HealthHistory {
attempts: attempts.to_vec(),
})
.is_some_and(|len| len <= RESPONSE_MAX_BYTES)
}
fn bound_events(events: &mut Vec<micromux::ServiceEvent>, truncated: &mut bool, keep_tail: bool) {
if events_response_fits(events, *truncated) {
return;
}
let original = std::mem::take(events);
let mut low = 0usize;
let mut high = original.len();
while low < high {
let middle = low + (high - low).div_ceil(2);
*events = if keep_tail {
original
.get(original.len().saturating_sub(middle)..)
.unwrap_or_default()
.to_vec()
} else {
original.get(..middle).unwrap_or_default().to_vec()
};
if events_response_fits(events, true) {
low = middle;
} else {
high = middle - 1;
}
}
*events = if keep_tail {
original
.get(original.len().saturating_sub(low)..)
.unwrap_or_default()
.to_vec()
} else {
original.get(..low).unwrap_or_default().to_vec()
};
*truncated = true;
}
fn events_response_fits(events: &[micromux::ServiceEvent], truncated: bool) -> bool {
encoded_len(&Response::Events {
events: events.to_vec(),
truncated,
})
.is_some_and(|len| len <= RESPONSE_MAX_BYTES)
}
fn rejection_response(rejection: CommandRejection) -> Response {
match rejection {
CommandRejection::UnknownService => {
Response::error(ErrorCode::UnknownService, "unknown service")
}
CommandRejection::InvalidState(message) => {
Response::error(ErrorCode::InvalidState, message)
}
CommandRejection::ConfigReload(message) => {
Response::error(ErrorCode::ConfigReload, message)
}
CommandRejection::PolicyDenied(message) => {
Response::error(ErrorCode::PolicyDenied, message)
}
CommandRejection::LimitExceeded(message) => {
Response::error(ErrorCode::LimitExceeded, message)
}
CommandRejection::RevisionMismatch { expected, actual } => Response::error(
ErrorCode::RevisionMismatch,
format!("expected revision {expected}, actual revision {actual}"),
),
CommandRejection::InvalidSpec(message) => Response::error(ErrorCode::InvalidSpec, message),
}
}
fn acknowledge_with<T>(
result: Result<Result<T, CommandRejection>, SchedulerStopped>,
ok: impl FnOnce(T) -> Response,
) -> Response {
match result {
Ok(Ok(value)) => ok(value),
Ok(Err(rejection)) => rejection_response(rejection),
Err(SchedulerStopped) => {
Response::error(ErrorCode::SchedulerStopped, "the scheduler has stopped")
}
}
}
fn acknowledge(result: Result<ServiceCommandResult, SchedulerStopped>) -> Response {
acknowledge_with(result, |services| Response::Accepted { services })
}
fn acknowledge_dynamic(
result: Result<micromux::DynamicServiceResult, SchedulerStopped>,
) -> Response {
acknowledge_with(result, Response::DynamicService)
}
fn acknowledge_reconcile(result: Result<micromux::ReconcileResult, SchedulerStopped>) -> Response {
acknowledge_with(result, Response::Reconcile)
}
#[cfg(test)]
mod tests {
use std::assert_matches;
use color_eyre::eyre;
use micromux::{
ChangeKind, HealthAttempt, HealthLine, LogLine, OutputStream, RestartPolicy, ServiceEvent,
ServiceEventKind, ServiceSnapshot, SessionChange,
};
use similar_asserts::assert_eq;
use crate::{
CanonicalConfigPath, ControlEndpoint, ControlServer, ErrorCode, PROTOCOL_VERSION, Request,
Response, ServiceBrief, SessionIdentity, SessionInfo,
};
use super::{
InboundFrameRate, MAX_LOG_TAIL, MAX_UNSOLICITED_SUBSCRIPTION_FRAMES_PER_WINDOW,
RESPONSE_MAX_BYTES, SubscriptionPolicy, bind_project, bound_follow_response_lines,
bound_follow_response_lines_page, bound_health_attempt, bound_health_history,
bound_tail_response_lines, claim_project_at, describe, dispatch, encoded_len,
lag_replay_changes, log_read_error, logs_fit, open_endpoint_lock, prune_dead_socket,
response_within_frame, stream_changes,
};
fn line(seq: u64, len: usize) -> LogLine {
LogLine {
seq,
run_generation: 1,
timestamp_unix_ms: 1_700_000_000_000 + seq,
line: "x".repeat(len),
}
}
fn snapshot(id: &str) -> ServiceSnapshot {
ServiceSnapshot::initial(
id.to_string(),
id.to_string(),
Vec::new(),
None,
RestartPolicy::Never,
vec!["true".to_string()],
None,
)
}
fn health_attempt(attempt: u64, output_bytes: usize) -> HealthAttempt {
HealthAttempt {
run_generation: 1,
attempt,
command: "probe".to_string(),
output: vec![HealthLine {
stream: OutputStream::Stdout,
line: "x".repeat(output_bytes),
}],
result: None,
}
}
#[test]
fn health_history_bounding_drops_oldest_attempts_before_trimming() {
let mut attempts = vec![
health_attempt(1, RESPONSE_MAX_BYTES / 2),
health_attempt(2, RESPONSE_MAX_BYTES / 2),
];
bound_health_history(&mut attempts);
assert_eq!(attempts.len(), 1);
assert_eq!(attempts.first().map(|attempt| attempt.attempt), Some(2));
assert!(
encoded_len(&Response::HealthHistory { attempts })
.is_some_and(|len| len <= RESPONSE_MAX_BYTES)
);
}
#[test]
fn health_history_bounding_trims_an_oversized_surviving_attempt() {
let mut attempts = vec![health_attempt(7, RESPONSE_MAX_BYTES + 100)];
bound_health_history(&mut attempts);
assert_eq!(attempts.first().map(|attempt| attempt.attempt), Some(7));
assert!(
encoded_len(&Response::HealthHistory { attempts })
.is_some_and(|len| len <= RESPONSE_MAX_BYTES)
);
}
#[test]
fn health_bounding_accounts_for_json_escaping() {
let escaped = "\\\"".repeat(1_300);
let mut attempt = Some(HealthAttempt {
run_generation: 1,
attempt: 7,
command: "probe".to_string(),
output: (0..200)
.map(|_| HealthLine {
stream: OutputStream::Stdout,
line: escaped.clone(),
})
.collect(),
result: None,
});
assert!(attempt.as_ref().is_some_and(|attempt| {
attempt
.output
.iter()
.map(|line| line.line.len())
.sum::<usize>()
< RESPONSE_MAX_BYTES
}));
bound_health_attempt(&mut attempt);
assert!(
encoded_len(&Response::Health(attempt)).is_some_and(|len| len <= RESPONSE_MAX_BYTES)
);
}
#[test]
fn event_bounding_keeps_the_requested_end_and_accounts_for_encoding() {
let escaped = "\\\"".repeat(1_300);
let events = (1..=200)
.map(|seq| ServiceEvent {
seq,
at_unix_ms: seq,
run_generation: 1,
kind: ServiceEventKind::Spawned,
detail: escaped.clone(),
exit_code: None,
pid: None,
delay_ms: None,
blocked_on: None,
})
.collect::<Vec<_>>();
assert!(events.iter().map(|event| event.detail.len()).sum::<usize>() < RESPONSE_MAX_BYTES);
let mut tail = events.clone();
let mut tail_truncated = false;
super::bound_events(&mut tail, &mut tail_truncated, true);
assert!(tail_truncated);
assert_eq!(tail.last().map(|event| event.seq), Some(200));
assert!(
encoded_len(&Response::Events {
events: tail,
truncated: tail_truncated,
})
.is_some_and(|len| len <= RESPONSE_MAX_BYTES)
);
let mut page = events;
let mut page_truncated = false;
super::bound_events(&mut page, &mut page_truncated, false);
assert!(page_truncated);
assert_eq!(page.first().map(|event| event.seq), Some(1));
assert!(
encoded_len(&Response::Events {
events: page,
truncated: page_truncated,
})
.is_some_and(|len| len <= RESPONSE_MAX_BYTES)
);
}
#[test]
fn lag_replay_invalidates_an_empty_roster() {
let changes = lag_replay_changes(&[]).collect::<Vec<_>>();
assert_eq!(changes.len(), 1);
assert_eq!(changes[0].service_id, SessionChange::SESSION_WIDE);
assert_eq!(changes[0].kind, ChangeKind::Roster);
}
#[test]
fn lag_replay_invalidates_the_roster_once_and_each_surviving_service() {
let snapshots = [snapshot("alpha"), snapshot("beta")];
let changes = lag_replay_changes(&snapshots)
.map(|change| (change.service_id, change.kind))
.collect::<Vec<_>>();
assert_eq!(
changes,
vec![
(SessionChange::SESSION_WIDE.to_string(), ChangeKind::Roster),
("alpha".to_string(), ChangeKind::Status),
("alpha".to_string(), ChangeKind::Logs),
("alpha".to_string(), ChangeKind::Health),
("alpha".to_string(), ChangeKind::Events),
("beta".to_string(), ChangeKind::Status),
("beta".to_string(), ChangeKind::Logs),
("beta".to_string(), ChangeKind::Health),
("beta".to_string(), ChangeKind::Events),
]
);
}
#[test]
fn oversized_unpageable_response_becomes_a_readable_limit_error() {
let response = Response::Services(vec![snapshot(&"\\\"".repeat(RESPONSE_MAX_BYTES))]);
let bounded = response_within_frame(&response);
assert_matches!(
bounded.as_ref(),
Response::Error {
code: ErrorCode::LimitExceeded,
..
}
);
assert!(encoded_len(bounded.as_ref()).is_some_and(|len| len <= crate::MAX_FRAME_BYTES));
}
#[test]
fn oversized_description_keeps_identity_and_marks_its_service_index() {
let services = (0..8)
.map(|index| ServiceBrief {
id: format!("service-{index}"),
name: "x".repeat(100 * 1024),
})
.collect::<Vec<_>>();
let info = SessionInfo {
session: None,
protocol_version: PROTOCOL_VERSION,
id: "session-id".to_string(),
pid: 42,
start_time: 7,
name: "session".to_string(),
working_dir: "/project".to_string(),
config_path: "/project/micromux.yaml".to_string(),
services,
services_truncated: false,
micromux_version: "test".to_string(),
capabilities: None,
};
let response = Response::Description(info.clone());
let bounded = response_within_frame(&response);
let Response::Description(bounded_info) = bounded.as_ref() else {
panic!("description should remain readable");
};
assert_eq!(bounded_info.id, info.id);
assert!(bounded_info.services_truncated);
assert!(bounded_info.services.len() < info.services.len());
assert!(encoded_len(bounded.as_ref()).is_some_and(|len| len <= RESPONSE_MAX_BYTES));
}
#[test]
fn tail_bounding_keeps_the_newest_lines() {
let mut lines = vec![
line(0, 200 * 1024),
line(1, 200 * 1024),
line(2, 200 * 1024),
line(3, 200 * 1024),
];
assert!(bound_tail_response_lines(&mut lines, None));
let seqs: Vec<u64> = lines.into_iter().map(|line| line.seq).collect();
assert_eq!(seqs, vec![2, 3]);
}
#[test]
fn follow_bounding_keeps_the_oldest_contiguous_page() {
let mut lines = vec![
line(0, 200 * 1024),
line(1, 200 * 1024),
line(2, 200 * 1024),
line(3, 200 * 1024),
];
assert!(bound_follow_response_lines(&mut lines, None));
let seqs: Vec<u64> = lines.into_iter().map(|line| line.seq).collect();
assert_eq!(seqs, vec![0, 1]);
}
#[test]
fn follow_bounding_trims_an_oversized_first_line_so_the_cursor_can_advance() {
let mut lines = vec![line(7, RESPONSE_MAX_BYTES + 10), line(8, 1)];
assert!(bound_follow_response_lines(&mut lines, None));
assert_eq!(lines.len(), 1);
assert_eq!(lines.first().map(|line| line.seq), Some(7));
assert!(logs_fit(&lines, true, None));
assert!(
lines
.first()
.is_some_and(|line| !line.line.is_empty() && line.line.len() < RESPONSE_MAX_BYTES)
);
}
#[test]
fn log_bounding_accounts_for_json_escape_expansion() {
let mut lines = vec![micromux::LogLine {
seq: 1,
run_generation: 1,
timestamp_unix_ms: 0,
line: "\0".repeat(RESPONSE_MAX_BYTES / 2),
}];
assert!(bound_follow_response_lines(&mut lines, None));
assert!(logs_fit(&lines, true, None));
assert_eq!(lines.first().map(|line| line.seq), Some(1));
}
#[tokio::test]
async fn project_lock_blocks_a_second_endpoint_for_the_same_config() -> eyre::Result<()> {
use std::os::unix::fs::PermissionsExt as _;
let directory = tempfile::tempdir()?;
let first = ControlEndpoint::Unix(directory.path().join("first.sock"));
let second = ControlEndpoint::Unix(directory.path().join("second.sock"));
let project_lock = directory.path().join("project.session.lock");
let claim = claim_project_at(&project_lock)?;
let held = claim_project_at(&project_lock);
assert!(
matches!(held, Err(crate::ClaimError::Held { pid: Some(pid) }) if pid == std::process::id()),
"{held:?}"
);
let first_guard = bind_project(&first, &claim)?.ok_or_else(|| eyre::eyre!("lock busy"))?;
drop(claim);
let ControlEndpoint::Unix(first_path) = &first else {
eyre::bail!("test endpoint was not Unix");
};
for lock_path in [first_path.with_extension("lock"), project_lock.clone()] {
assert_eq!(
std::fs::metadata(lock_path)?.permissions().mode() & 0o777,
0o600
);
}
assert!(matches!(
claim_project_at(&project_lock),
Err(crate::ClaimError::Held { .. })
));
drop(first_guard);
assert!(bind_project(&second, &claim_project_at(&project_lock)?)?.is_some());
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn pruning_never_removes_or_blocks_a_bound_endpoint() -> eyre::Result<()> {
use std::os::unix::fs::FileTypeExt as _;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
const BINDS: usize = 300;
let directory = tempfile::tempdir()?;
let socket_path = directory.path().join("raced.sock");
let stop = Arc::new(AtomicBool::new(false));
let removed = Arc::new(AtomicUsize::new(0));
let pruning = std::thread::spawn({
let socket_path = socket_path.clone();
let stop = Arc::clone(&stop);
let removed = Arc::clone(&removed);
move || -> std::io::Result<()> {
while !stop.load(Ordering::Relaxed) {
if prune_dead_socket(&socket_path)? {
removed.fetch_add(1, Ordering::Relaxed);
}
}
Ok(())
}
});
let binding = tokio::task::spawn_blocking({
let socket_path = socket_path.clone();
move || -> eyre::Result<()> {
let endpoint = ControlEndpoint::Unix(socket_path.clone());
for attempt in 0..BINDS {
let guard = super::bind(&endpoint)?
.ok_or_else(|| eyre::eyre!("bind {attempt} read the pruner as an owner"))?;
std::thread::sleep(std::time::Duration::from_micros(200));
let bound = std::fs::symlink_metadata(&socket_path)
.is_ok_and(|metadata| metadata.file_type().is_socket());
if !bound {
eyre::bail!("bind {attempt} lost its socket to the pruner");
}
drop(guard);
}
Ok(())
}
});
let binding_result = binding.await;
stop.store(true, Ordering::Relaxed);
let pruning_result = pruning
.join()
.map_err(|_| eyre::eyre!("pruning thread panicked"))?;
binding_result??;
pruning_result?;
assert_eq!(removed.load(Ordering::Relaxed), 0);
Ok(())
}
struct UnscheduledSession {
server: ControlServer,
terminal: micromux::TerminalControl,
runner: std::pin::Pin<Box<dyn Future<Output = Result<(), micromux::Error>>>>,
}
fn unscheduled_session(directory: &std::path::Path) -> eyre::Result<UnscheduledSession> {
use std::sync::Arc;
let yaml = "version: 1\nservices:\n svc:\n command: [\"true\"]\n";
let config_path = directory.join("micromux.yaml");
std::fs::write(&config_path, yaml)?;
let config_path = CanonicalConfigPath::new(config_path)?;
let mut diagnostics = Vec::new();
let config = micromux::from_str(yaml, directory, 0usize, None, &mut diagnostics)?;
let mux = Arc::new(micromux::Micromux::new(&config)?);
let (runner, handles) = mux.start(micromux::CancellationToken::new());
let control = handles.service_control();
let server = ControlServer::new(
handles.reader,
control,
SessionIdentity::new("test".to_string(), directory, &config_path),
handles.dynamic_services,
);
Ok(UnscheduledSession {
server,
terminal: handles.terminal,
runner: Box::pin(runner),
})
}
#[tokio::test]
async fn terminal_requests_need_a_terminal_capability() -> eyre::Result<()> {
let directory = tempfile::tempdir()?;
let UnscheduledSession { server, .. } = unscheduled_session(directory.path())?;
for request in [
Request::SendInput {
service: "svc".to_string(),
data: crate::InputBytes::from(b"x".to_vec()),
paste: false,
},
Request::ResizeTerminal { cols: 80, rows: 24 },
] {
assert_matches!(
dispatch(&server, request).await,
Response::Error {
code: ErrorCode::PolicyDenied,
..
}
);
}
Ok(())
}
#[tokio::test]
async fn terminal_requests_are_validated_before_queueing() -> eyre::Result<()> {
let directory = tempfile::tempdir()?;
let UnscheduledSession {
server,
terminal,
runner,
} = unscheduled_session(directory.path())?;
let server = server.with_terminal(terminal);
let send = |service: &str| Request::SendInput {
service: service.to_string(),
data: crate::InputBytes::from(b"x".to_vec()),
paste: false,
};
assert_matches!(
dispatch(&server, send("missing")).await,
Response::Error {
code: ErrorCode::UnknownService,
..
}
);
assert_matches!(
dispatch(&server, send("svc")).await,
Response::Error {
code: ErrorCode::InvalidState,
message,
} if message == "terminal input discarded: svc is not running"
);
for (cols, rows) in [(0, 24), (80, 0)] {
assert_matches!(
dispatch(&server, Request::ResizeTerminal { cols, rows }).await,
Response::Error {
code: ErrorCode::BadRequest,
..
}
);
}
assert_matches!(
dispatch(&server, Request::ResizeTerminal { cols: 80, rows: 24 }).await,
Response::TerminalQueued
);
drop(runner);
assert_matches!(
dispatch(&server, Request::ResizeTerminal { cols: 80, rows: 24 }).await,
Response::Error {
code: ErrorCode::SchedulerStopped,
..
}
);
Ok(())
}
#[test]
fn pruning_removes_only_sockets_without_an_owner() -> eyre::Result<()> {
let directory = tempfile::tempdir()?;
let leaked = directory.path().join("leaked.sock");
drop(std::os::unix::net::UnixListener::bind(&leaked)?);
assert!(prune_dead_socket(&leaked)?);
assert!(!leaked.exists());
assert!(leaked.with_extension("lock").exists());
let starting = directory.path().join("starting.sock");
drop(std::os::unix::net::UnixListener::bind(&starting)?);
let owner = open_endpoint_lock(&starting)?;
fs2::FileExt::try_lock_exclusive(&owner)?;
assert!(!prune_dead_socket(&starting)?);
assert!(starting.exists());
drop(owner);
assert!(prune_dead_socket(&starting)?);
assert!(!starting.exists());
Ok(())
}
#[tokio::test]
async fn subscription_keepalives_and_rate_limits_client_frames() -> eyre::Result<()> {
use std::sync::Arc;
use std::time::Duration;
let directory = tempfile::tempdir()?;
let yaml = "version: 1\nservices: {}\n";
let config_path = directory.path().join("micromux.yaml");
std::fs::write(&config_path, yaml)?;
let config_path = CanonicalConfigPath::new(config_path)?;
let mut diagnostics = Vec::new();
let config = micromux::from_str(yaml, directory.path(), 0usize, None, &mut diagnostics)?;
let mux = Arc::new(
micromux::Micromux::new(&config)?.with_cache_directory(directory.path().join("cache")),
);
let (_runner, handles) = mux.start(micromux::CancellationToken::new());
let control = handles.service_control();
let server = Arc::new(ControlServer::new(
handles.reader,
control,
SessionIdentity::new("test".to_string(), directory.path(), &config_path),
handles.dynamic_services,
));
let disk_log_reads = describe(&server)
.capabilities
.and_then(|capabilities| capabilities.disk_log_reads)
.ok_or_else(|| eyre::eyre!("describe omitted disk-log pool health"))?;
assert!(disk_log_reads.available);
assert_eq!(disk_log_reads.workers, 4);
let (server_io, client_io) = tokio::io::duplex(4096);
let shutdown = micromux::CancellationToken::new();
let policy = SubscriptionPolicy {
keepalive_interval: Duration::from_millis(40),
keepalive_kind: ChangeKind::Heartbeat,
inbound_frame_window: Duration::from_millis(80),
};
let task = tokio::spawn({
let server = Arc::clone(&server);
let shutdown = shutdown.clone();
async move {
stream_changes(&server, crate::framed(server_io), shutdown, policy).await;
}
});
let mut client = crate::framed(client_io);
crate::write_message(&mut client, &Request::Describe).await?;
let response = tokio::time::timeout(
Duration::from_secs(1),
crate::read_message::<_, Response>(&mut client),
)
.await??
.ok_or_else(|| eyre::eyre!("subscription closed before its heartbeat"))?;
assert_matches!(
response,
Response::Change(SessionChange {
service_id,
kind: ChangeKind::Heartbeat,
}) if service_id == SessionChange::SESSION_WIDE
);
assert!(!task.is_finished());
let mut sent_frames = 0usize;
for _ in 0..=MAX_UNSOLICITED_SUBSCRIPTION_FRAMES_PER_WINDOW {
match crate::write_message(&mut client, &Request::Describe).await {
Ok(()) => sent_frames = sent_frames.saturating_add(1),
Err(_) if sent_frames >= MAX_UNSOLICITED_SUBSCRIPTION_FRAMES_PER_WINDOW => break,
Err(err) => return Err(err.into()),
}
}
tokio::time::timeout(Duration::from_secs(1), task).await??;
let (server_io, client_io) = tokio::io::duplex(4096);
let legacy_policy = SubscriptionPolicy {
keepalive_interval: Duration::from_millis(20),
keepalive_kind: ChangeKind::Events,
inbound_frame_window: Duration::from_secs(1),
};
let legacy_task = tokio::spawn({
let server = Arc::clone(&server);
let shutdown = shutdown.clone();
async move {
stream_changes(&server, crate::framed(server_io), shutdown, legacy_policy).await;
}
});
let mut legacy_client = crate::framed(client_io);
let legacy_response = tokio::time::timeout(
Duration::from_secs(1),
crate::read_message::<_, Response>(&mut legacy_client),
)
.await??
.ok_or_else(|| eyre::eyre!("legacy subscription closed before its liveness write"))?;
assert_matches!(
legacy_response,
Response::Change(SessionChange {
service_id,
kind: ChangeKind::Events,
}) if service_id == SessionChange::SESSION_WIDE
);
assert!(!legacy_task.is_finished());
shutdown.cancel();
tokio::time::timeout(Duration::from_secs(1), legacy_task).await??;
Ok(())
}
#[test]
fn subscription_frame_allowance_replenishes_after_its_rate_window() {
let started = tokio::time::Instant::now();
let window = std::time::Duration::from_secs(1);
let mut rate = InboundFrameRate::new();
for _ in 0..MAX_UNSOLICITED_SUBSCRIPTION_FRAMES_PER_WINDOW {
assert!(rate.accepts(started, window));
}
assert!(!rate.accepts(started, window));
assert!(rate.accepts(started + window, window));
}
#[test]
fn subscription_frame_rate_has_no_fixed_window_boundary_burst() {
let started = tokio::time::Instant::now();
let window = std::time::Duration::from_secs(1);
let halfway = MAX_UNSOLICITED_SUBSCRIPTION_FRAMES_PER_WINDOW / 2;
let mut rate = InboundFrameRate::new();
for _ in 0..halfway {
assert!(rate.accepts(started, window));
}
let just_before_window = started + window - std::time::Duration::from_nanos(1);
for _ in halfway..MAX_UNSOLICITED_SUBSCRIPTION_FRAMES_PER_WINDOW {
assert!(rate.accepts(just_before_window, window));
}
let boundary = started + window;
for _ in 0..halfway {
assert!(rate.accepts(boundary, window));
}
assert!(!rate.accepts(boundary, window));
}
#[test]
fn follow_page_reports_truncated_when_a_sentinel_line_was_fetched() {
fn seq(value: usize) -> u64 {
u64::try_from(value).unwrap_or(u64::MAX)
}
let lines = (0..=MAX_LOG_TAIL)
.map(|value| line(seq(value), 1))
.collect::<Vec<_>>();
let (lines, truncated) = bound_follow_response_lines_page(lines, None);
assert!(truncated);
assert_eq!(lines.len(), MAX_LOG_TAIL);
assert_eq!(lines.first().map(|line| line.seq), Some(0));
assert_eq!(
lines.last().map(|line| line.seq),
Some(seq(MAX_LOG_TAIL.saturating_sub(1)))
);
}
#[test]
fn rotating_run_log_is_reported_as_retryable_busy() {
assert_matches!(
log_read_error(micromux::LogRunReadError::Rotating),
Response::Error {
code: ErrorCode::Busy,
message,
} if message.contains("rotating")
);
}
}