use super::{SupervisedModule, SupervisorHandle, SupervisorSnapshot};
#[cfg(test)]
pub(super) type ResourceCommitPause = (
tokio::sync::oneshot::Sender<()>,
std::sync::mpsc::Receiver<()>,
);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum ResourceCommitError {
Busy,
Changed,
}
#[derive(Debug, Clone)]
pub(crate) struct ResourceLaunch {
pub pid: Option<u32>,
pub start: Option<u64>,
pub spawn_generation: u64,
pub enabled: bool,
pub alive: bool,
pub target: crate::subtree_resources::Target,
}
impl ResourceLaunch {
fn from_snapshot(snapshot: &SupervisorSnapshot) -> Self {
Self {
pid: snapshot.reported_pid(),
start: snapshot.process_start_time,
spawn_generation: snapshot.spawn_generation,
enabled: snapshot.enabled,
alive: snapshot.process_alive,
target: snapshot.subtree_target.clone(),
}
}
pub(crate) fn generation(&self) -> u128 {
(u128::from(self.spawn_generation) << 64) | u128::from(self.start.unwrap_or(0))
}
fn same_launch(&self, other: &Self) -> bool {
self.pid == other.pid
&& self.start == other.start
&& self.spawn_generation == other.spawn_generation
&& self.enabled == other.enabled
&& self.alive == other.alive
}
}
impl SupervisedModule {
#[cfg(test)]
pub(crate) fn with_resource_snapshot_locked_for_test<T>(&self, work: impl FnOnce() -> T) -> T {
let _snapshot = self.inner.snapshot.lock().unwrap();
work()
}
#[cfg(all(test, unix, feature = "test-support"))]
pub(crate) fn set_launch_alive_for_test(&self, alive: bool) {
self.inner.snapshot.lock().unwrap().process_alive = alive;
}
#[cfg(all(test, unix, feature = "test-support"))]
pub(crate) fn set_launch_exited_predecessor_for_test(&self, pid: u32) {
let mut snapshot = self.inner.snapshot.lock().unwrap();
snapshot.pid = Some(pid);
snapshot.process_alive = false;
}
pub(crate) fn resource_launch(&self) -> Option<ResourceLaunch> {
let snapshot = self.inner.snapshot.try_lock().ok()?;
Some(ResourceLaunch::from_snapshot(&snapshot))
}
}
impl SupervisorHandle {
#[cfg(test)]
pub(crate) fn resource_commit_pause_armed_for_test(&self) -> bool {
self.resource_commit_pause.lock().unwrap().is_some()
}
#[cfg(all(test, unix))]
pub(crate) fn pause_resource_commit_for_test(
&self,
) -> (
tokio::sync::oneshot::Receiver<()>,
std::sync::mpsc::Sender<()>,
) {
let (reached, reached_rx) = tokio::sync::oneshot::channel();
let (resume, resume_rx) = std::sync::mpsc::channel();
*self.resource_commit_pause.lock().unwrap() = Some((reached, resume_rx));
(reached_rx, resume)
}
pub(crate) fn with_current_resource_launch<T>(
&self,
id: &str,
nonce: &str,
launch: &ResourceLaunch,
commit: impl FnOnce() -> T,
) -> Result<T, ResourceCommitError> {
#[cfg(test)]
{
let pause = self.resource_commit_pause.lock().unwrap().take();
if let Some((reached, resume)) = pause {
let _hold = crate::config_gate::runtime::commit_pause::Hold::current();
let _ = reached.send(());
let _ = resume.recv();
}
}
let runtime = self
.gate_runtime
.try_lock()
.map_err(|_| ResourceCommitError::Busy)?
.clone();
if let Some(runtime) = runtime {
let owner = runtime.upgrade().ok_or(ResourceCommitError::Changed)?;
return owner
.with_legacy_nonce_authority(id, || {
self.commit_current_resource_launch(id, nonce, launch, commit)
})
.map_err(|()| ResourceCommitError::Busy)?
.ok_or(ResourceCommitError::Changed)?;
}
self.commit_current_resource_launch(id, nonce, launch, commit)
}
pub(crate) fn channel_launch_stopped(&self, spawn: &super::RegistrationSpawn) -> Option<bool> {
let module = self.get(&spawn.module_id)?;
let launch = module.resource_launch()?;
let identity = spawn.identity.get()?;
(launch.pid == Some(identity.pid) && launch.start == Some(identity.start_time))
.then_some(!launch.enabled || !launch.alive)
}
pub(crate) fn with_current_channel_resource_launch<T>(
&self,
registry: &crate::Registry,
connection: crate::ConnectionId,
spawn: &super::RegistrationSpawn,
launch: &ResourceLaunch,
commit: impl FnOnce() -> T,
) -> Result<T, ResourceCommitError> {
#[cfg(test)]
{
let pause = self.resource_commit_pause.lock().unwrap().take();
if let Some((reached, resume)) = pause {
let _hold = crate::config_gate::runtime::commit_pause::Hold::current();
let _ = reached.send(());
let _ = resume.recv();
}
}
let id = spawn.module_id.as_str();
if self.module_is_gated(id) && !spawn.has_program_binding() {
return Err(ResourceCommitError::Changed);
}
let _alive = spawn
.begin_publication()
.map_err(|_| ResourceCommitError::Changed)?;
let modules = self
.modules
.try_lock()
.map_err(|_| ResourceCommitError::Busy)?;
let module = modules.get(id).ok_or(ResourceCommitError::Changed)?;
let snapshot = module
.inner
.snapshot
.try_lock()
.map_err(|_| ResourceCommitError::Busy)?;
let current = ResourceLaunch::from_snapshot(&snapshot);
let serving = registry
.get_module(id)
.ok()
.flatten()
.is_some_and(|registration| registration.connection_id == connection);
let same_process = spawn.identity.get().is_some_and(|identity| {
current.pid == Some(identity.pid) && current.start == Some(identity.start_time)
});
if !serving || !same_process || !launch.same_launch(¤t) {
return Err(ResourceCommitError::Changed);
}
Ok(commit())
}
fn commit_current_resource_launch<T>(
&self,
id: &str,
nonce: &str,
launch: &ResourceLaunch,
commit: impl FnOnce() -> T,
) -> Result<T, ResourceCommitError> {
let modules = self
.modules
.try_lock()
.map_err(|_| ResourceCommitError::Busy)?;
let nonces = self
.spawn_nonces
.try_lock()
.map_err(|_| ResourceCommitError::Busy)?;
if nonces.get(id).map(String::as_str) != Some(nonce) {
return Err(ResourceCommitError::Changed);
}
let module = modules.get(id).ok_or(ResourceCommitError::Changed)?;
let snapshot = module
.inner
.snapshot
.try_lock()
.map_err(|_| ResourceCommitError::Busy)?;
if !launch.same_launch(&ResourceLaunch::from_snapshot(&snapshot)) {
return Err(ResourceCommitError::Changed);
}
Ok(commit())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn launch_snapshot_is_atomic_and_a_stale_declaration_cannot_commit() {
let handle = SupervisorHandle::new();
let supervisor = super::super::Supervisor::new_for_test(
std::sync::Arc::new(crate::Registry::default()),
super::super::RestartPolicy::default(),
)
.with_handle(handle.clone());
let module = supervisor
.supervise_configured(
crate::ModuleSpec {
script: None,
module_id: "resource-launch-test".into(),
program: std::env::current_exe().unwrap(),
args: Vec::new(),
env: Vec::new(),
reserved: false,
reserved_prefixes: Vec::new(),
protocol: subc_control::ModuleProtocol::None,
overlap: Default::default(),
carry_containment: false,
},
false,
)
.unwrap();
{
let mut snapshot = module.inner.snapshot.lock().unwrap();
snapshot.pid = Some(42);
snapshot.process_start_time = Some(100);
snapshot.spawn_generation = 7;
snapshot.enabled = true;
snapshot.process_alive = true;
assert!(
module.resource_launch().is_none(),
"contention cannot be mistaken for a missing container"
);
}
let launch = module.resource_launch().unwrap();
assert_eq!(launch.generation(), (7_u128 << 64) | 100);
let id = module.module_id();
handle
.spawn_nonces
.lock()
.unwrap()
.insert(id.into(), "nonce".into());
assert_eq!(
handle.with_current_resource_launch(id, "nonce", &launch, || 123),
Ok(123)
);
macro_rules! assert_busy {
($lock:expr) => {{
let guard = $lock;
assert_eq!(
handle.with_current_resource_launch(id, "nonce", &launch, || {
panic!("a contended commit fence must not run the mutation")
}),
Err::<(), _>(ResourceCommitError::Busy)
);
drop(guard);
}};
}
assert_busy!(handle.modules.lock().unwrap());
assert_busy!(handle.spawn_nonces.lock().unwrap());
assert_busy!(module.inner.snapshot.lock().unwrap());
module.inner.snapshot.lock().unwrap().spawn_generation += 1;
assert!(handle
.with_current_resource_launch(id, "nonce", &launch, || panic!(
"stale generation committed"
))
.is_err_and(|error| error == ResourceCommitError::Changed));
let current = module.resource_launch().unwrap();
handle
.spawn_nonces
.lock()
.unwrap()
.insert(id.into(), "replacement".into());
assert!(handle
.with_current_resource_launch(id, "nonce", ¤t, || panic!("stale nonce committed"))
.is_err_and(|error| error == ResourceCommitError::Changed));
}
#[tokio::test]
async fn channel_fence_commits_only_for_the_live_serving_spawn() {
use super::super::{ProcessIdentity, RegistrationSpawn, SpawnRole};
let handle = SupervisorHandle::new();
let supervisor = super::super::Supervisor::new_for_test(
std::sync::Arc::new(crate::Registry::default()),
super::super::RestartPolicy::default(),
)
.with_handle(handle.clone());
let module = supervisor
.supervise_configured(
crate::ModuleSpec {
script: None,
module_id: "channel-fence-test".into(),
program: std::env::current_exe().unwrap(),
args: Vec::new(),
env: Vec::new(),
reserved: false,
reserved_prefixes: Vec::new(),
protocol: subc_control::ModuleProtocol::None,
overlap: Default::default(),
carry_containment: false,
},
false,
)
.unwrap();
let set_process = |pid: u32, start: u64| {
let mut snapshot = module.inner.snapshot.lock().unwrap();
snapshot.pid = Some(pid);
snapshot.process_start_time = Some(start);
snapshot.spawn_generation = 7;
snapshot.enabled = true;
snapshot.process_alive = true;
};
set_process(42, 100);
let id = module.module_id();
let registry = crate::Registry::default();
let serving = crate::ConnectionId::new(5);
registry
.register_with_control_ops(
subc_protocol::manifest::ModuleManifest::builder(id, "1.0.0").build(),
subc_protocol::PROTOCOL_VERSION,
serving,
Vec::new(),
)
.unwrap();
let live_spawn = |pid: u32, start: u64| {
let spawn = RegistrationSpawn::new(&handle, id, SpawnRole::Plain, None);
spawn
.identity
.set(ProcessIdentity {
pid,
start_time: start,
})
.unwrap();
spawn.ready();
spawn
};
let spawn = live_spawn(42, 100);
let launch = module.resource_launch().unwrap();
let fence = |connection, spawn: &RegistrationSpawn, launch: &ResourceLaunch| {
handle.with_current_channel_resource_launch(®istry, connection, spawn, launch, || 1)
};
assert_eq!(fence(serving, &spawn, &launch), Ok(1));
assert_eq!(
fence(crate::ConnectionId::new(6), &spawn, &launch),
Err(ResourceCommitError::Changed),
"a connection the registry does not name as serving cannot commit"
);
let retiring = spawn.clone();
let retirement = handle
.with_current_channel_resource_launch(®istry, serving, &spawn, &launch, || {
let (starting, started) = std::sync::mpsc::channel();
let retirement = std::thread::spawn(move || {
starting.send(()).unwrap();
retiring.retire();
});
started.recv().unwrap();
std::thread::sleep(std::time::Duration::from_millis(50));
assert!(
!retirement.is_finished(),
"retirement ran inside the commit"
);
assert!(spawn.launch_ready(), "retirement ran inside the commit");
retirement
})
.unwrap();
retirement.join().unwrap();
assert!(!spawn.launch_ready());
assert_eq!(
fence(serving, &spawn, &launch),
Err(ResourceCommitError::Changed),
"a retired spawn cannot commit"
);
let superseded = live_spawn(42, 100);
set_process(43, 200);
let current = module.resource_launch().unwrap();
assert_eq!(
fence(serving, &superseded, ¤t),
Err(ResourceCommitError::Changed)
);
assert_eq!(fence(serving, &live_spawn(43, 200), ¤t), Ok(1));
let incumbent = live_spawn(43, 200);
let rewrite = handle
.with_current_channel_resource_launch(®istry, serving, &incumbent, ¤t, || {
registry
.register_candidate_with_control_ops(
subc_protocol::manifest::ModuleManifest::builder(id, "1.0.0").build(),
subc_protocol::PROTOCOL_VERSION,
crate::ConnectionId::new(7),
Vec::new(),
)
.unwrap();
registry.promote_candidate(id).unwrap().unwrap();
let rewriting = module.clone();
let (starting, started) = std::sync::mpsc::channel();
let rewrite = std::thread::spawn(move || {
starting.send(()).unwrap();
rewriting.inner.snapshot.lock().unwrap().pid = Some(44);
});
started.recv().unwrap();
std::thread::sleep(std::time::Duration::from_millis(50));
assert!(
!rewrite.is_finished(),
"the launch record was rewritten inside the commit"
);
rewrite
})
.unwrap();
rewrite.join().unwrap();
assert_eq!(
fence(serving, &incumbent, ¤t),
Err(ResourceCommitError::Changed)
);
let unpromoted = crate::Registry::default();
unpromoted
.register_with_control_ops(
subc_protocol::manifest::ModuleManifest::builder(id, "1.0.0").build(),
subc_protocol::PROTOCOL_VERSION,
serving,
Vec::new(),
)
.unwrap();
let rewritten = module.resource_launch().unwrap();
assert_eq!(
handle.with_current_channel_resource_launch(
&unpromoted,
serving,
&incumbent,
&rewritten,
|| 1
),
Err(ResourceCommitError::Changed)
);
set_process(43, 200);
let current = module.resource_launch().unwrap();
let fresh = live_spawn(43, 200);
let _snapshot = module.inner.snapshot.lock().unwrap();
assert_eq!(
fence(serving, &fresh, ¤t),
Err(ResourceCommitError::Busy)
);
}
}