use std::{
collections::{
HashMap,
HashSet,
},
sync::{
Arc,
Mutex,
Weak,
atomic::{
AtomicU16,
AtomicU64,
Ordering,
},
},
time::{
Duration,
SystemTime,
},
};
use tokio::{
sync::{
broadcast,
mpsc,
},
task::AbortHandle,
};
use crate::{
config::CasConfig,
cw::CwAdvance,
ecmg::driver::{
DriverConfig,
DriverEvent,
DriverHandle,
DriverState,
},
emmg::{
Dispatch,
EmmEvent,
EmmgListeners,
},
error::Error,
event::{
CasError,
CasParams,
CasState,
CasStatus,
Event,
StreamInfo,
StreamStatus,
},
lock_ext::MutexExt,
service::{
Member,
ServiceGroup,
},
simulcrypt::CasRef,
spawn::spawn_supervised,
};
#[derive(Clone)]
pub struct StreamMembership {
pub stream_id: u16,
pub group: Weak<ServiceGroup>,
}
pub struct CasEntry {
settings: Mutex<CasSettings>,
driver: Mutex<Option<DriverHandle>>,
generation: AtomicU64,
state: Mutex<CasState>,
used_ecm_pids: Mutex<HashMap<u16, StreamMembership>>,
next_stream_offset: AtomicU16,
stream_sinks: Mutex<HashMap<u16, StreamSlot>>,
connects: AtomicU64,
last_error: Mutex<Option<CasError>>,
ready_streams: Mutex<HashSet<u16>>,
pending_add_stream: Mutex<HashMap<u16, AbortHandle>>,
}
struct CasSettings {
config: CasConfig,
emmg: Option<Dispatch>,
}
impl CasSettings {
fn emmg_channel(&self) -> Option<(Dispatch, u16)> {
self.emmg
.clone()
.map(|dispatch| (dispatch, self.config.ecmg.channel_id))
}
}
fn warn_if_below_min_cp(
name: &str,
stream_id: u16,
nominal_cp_duration: u16,
min_cp_duration: Duration,
) {
let crypto_period = Duration::from_millis(u64::from(nominal_cp_duration) * 100);
if crypto_period < min_cp_duration {
tracing::warn!(
name,
stream_id,
crypto_period = ?crypto_period,
min_cp_duration = ?min_cp_duration,
"crypto_period is shorter than the ECMG min_CP_duration"
);
}
}
fn driver_config(cfg: &CasConfig) -> DriverConfig {
DriverConfig {
name: cfg.name.clone(),
channel_id: cfg.ecmg.channel_id,
super_cas_id: cfg.super_cas_id,
addr: cfg.ecmg.addr,
reconnect_delay: cfg.ecmg.reconnect_delay,
response_timeout: cfg.ecmg.response_timeout,
idle_probe_interval: cfg.ecmg.idle_probe_interval,
}
}
impl CasEntry {
pub fn new(config: CasConfig, emmg: Option<Dispatch>) -> Arc<Self> {
Arc::new(Self {
settings: Mutex::new(CasSettings { config, emmg }),
driver: Mutex::new(None),
generation: AtomicU64::new(0),
state: Mutex::new(CasState::Connecting),
used_ecm_pids: Mutex::new(HashMap::new()),
next_stream_offset: AtomicU16::new(0),
stream_sinks: Mutex::new(HashMap::new()),
connects: AtomicU64::new(0),
last_error: Mutex::new(None),
ready_streams: Mutex::new(HashSet::new()),
pending_add_stream: Mutex::new(HashMap::new()),
})
}
pub fn ensure_running(self: &Arc<Self>, cas: CasRef) {
let mut driver_slot = self.driver.lock_or_recover();
if driver_slot.is_some() {
return;
}
let driver_config = driver_config(&self.settings.lock_or_recover().config);
let generation = self.generation.fetch_add(1, Ordering::Relaxed) + 1;
let (events_tx, events_rx) = mpsc::unbounded_channel();
let handle = DriverHandle::spawn(driver_config, events_tx);
spawn_supervised(
"forward-driver-events",
Arc::clone(self).forward_driver_events(cas, generation, events_rx),
);
*self.state.lock_or_recover() = CasState::Connecting;
*driver_slot = Some(handle);
}
fn shutdown(&self) {
for (_, handle) in self.pending_add_stream.lock_or_recover().drain() {
handle.abort();
}
self.driver.lock_or_recover().take();
self.generation.fetch_add(1, Ordering::Relaxed);
self.ready_streams.lock_or_recover().clear();
*self.state.lock_or_recover() = CasState::Disconnected;
}
pub fn update(
self: &Arc<Self>,
listeners: &EmmgListeners,
cas: CasRef,
cfg: CasConfig,
) -> Result<(), Error> {
let (old, old_emmg) = {
let settings = self.settings.lock_or_recover();
(settings.config.clone(), settings.emmg.clone())
};
if old == cfg {
return Ok(());
}
let old_port = old.emmg.as_ref().map(|emmg| emmg.port);
let new_port = cfg.emmg.as_ref().map(|emmg| emmg.port);
let emmg = if old_port.is_some()
&& old_port == new_port
&& old.ecmg.channel_id == cfg.ecmg.channel_id
{
if let (Some(dispatch), Some(emmg)) = (&old_emmg, &cfg.emmg) {
listeners.ensure_udp(emmg)?;
dispatch.update_channel(cfg.ecmg.channel_id, cfg.super_cas_id, emmg.broadcast);
}
old_emmg
} else {
let new_emmg = match &cfg.emmg {
Some(emmg) => Some(listeners.register(&cfg, emmg)?),
None => None,
};
if let Some(dispatch) = &new_emmg {
let sinks: Vec<_> = self
.stream_sinks
.lock_or_recover()
.iter()
.map(|(stream_id, slot)| (*stream_id, slot.clone()))
.collect();
for (stream_id, slot) in sinks {
register_emmg_stream(
cas,
&cfg.name,
dispatch,
cfg.ecmg.channel_id,
stream_id,
&slot,
);
}
}
if let (Some(dispatch), Some(port)) = (old_emmg, old_port) {
listeners.unregister(&dispatch, port, old.ecmg.channel_id);
}
new_emmg
};
let restart =
(&old.name, old.super_cas_id, &old.ecmg) != (&cfg.name, cfg.super_cas_id, &cfg.ecmg);
*self.settings.lock_or_recover() = CasSettings { config: cfg, emmg };
if restart && self.driver.lock_or_recover().is_some() {
self.shutdown();
self.ensure_running(cas);
let streams: Vec<_> = self
.used_ecm_pids
.lock_or_recover()
.values()
.filter_map(|membership| {
let group = membership.group.upgrade()?;
Some((membership.stream_id, group.nominal_cp_duration()))
})
.collect();
for (stream_id, nominal_cp_duration) in streams {
add_ecmg_stream(self, stream_id, nominal_cp_duration);
}
}
Ok(())
}
fn check_min_cp_duration(&self, min_cp_duration: Duration) {
let name = self.settings.lock_or_recover().config.name.clone();
for membership in self.used_ecm_pids.lock_or_recover().values() {
if let Some(group) = membership.group.upgrade() {
warn_if_below_min_cp(
&name,
membership.stream_id,
group.nominal_cp_duration(),
min_cp_duration,
);
}
}
}
pub fn status(&self, cas: CasRef) -> CasStatus {
let memberships: Vec<_> = self
.used_ecm_pids
.lock_or_recover()
.iter()
.map(|(ecm_pid, membership)| (*ecm_pid, membership.clone()))
.collect();
let mut streams: Vec<StreamStatus> = memberships
.into_iter()
.map(|(ecm_pid, membership)| {
let stream_id = membership.stream_id;
let counters = self
.stream_sinks
.lock_or_recover()
.get(&stream_id)
.map(|slot| Arc::clone(&slot.counters))
.unwrap_or_default();
StreamStatus {
stream_id,
ecm_pid,
ready: self.ready_streams.lock_or_recover().contains(&stream_id),
cp_number: membership
.group
.upgrade()
.and_then(|group| group.cp_number()),
ecm: counters.ecm.load(Ordering::Relaxed),
ecm_stale: counters.ecm_stale.load(Ordering::Relaxed),
emm: counters.emm.load(Ordering::Relaxed),
}
})
.collect();
streams.sort_by_key(|stream| stream.stream_id);
CasStatus {
cas,
name: self.settings.lock_or_recover().config.name.clone(),
state: self.state.lock_or_recover().clone(),
last_error: self.last_error.lock_or_recover().clone(),
connects: self.connects.load(Ordering::Relaxed),
streams,
}
}
pub fn detach(&self, emmg: &EmmgListeners, cas: CasRef) {
self.shutdown();
let settings = self.settings.lock_or_recover();
if let (Some(dispatch), Some(emmg_cfg)) = (&settings.emmg, &settings.config.emmg) {
emmg.unregister(dispatch, emmg_cfg.port, settings.config.ecmg.channel_id);
}
drop(settings);
for (_, membership) in self.used_ecm_pids.lock_or_recover().drain() {
if let Some(group) = membership.group.upgrade() {
group.remove_member(cas, membership.stream_id);
}
}
for (_, slot) in self.stream_sinks.lock_or_recover().drain() {
let _ = slot.sink.send(Event::State {
cas,
state: CasState::Disconnected,
});
}
}
pub fn membership(&self, ecm_pid: u16) -> Option<StreamMembership> {
self.used_ecm_pids.lock_or_recover().get(&ecm_pid).cloned()
}
pub fn is_stream_ready(&self, stream_id: u16) -> bool {
self.ready_streams.lock_or_recover().contains(&stream_id)
}
pub fn provision_cw(&self, member: Member, advance: &CwAdvance) {
let Some(driver) = self.driver.lock_or_recover().clone() else {
return;
};
let access_criteria =
(!member.access_criteria.is_empty()).then_some(member.access_criteria);
if let Err(error) = driver.provision_cw(
member.stream_id,
advance.cp_number,
advance.current,
advance.next,
access_criteria,
) {
tracing::warn!(
name = %self.settings.lock_or_recover().config.name,
stream_id = member.stream_id,
error = %error,
"failed to queue CW_PROVISION"
);
}
}
pub fn attach_stream(&self, ecm_pid: u16, group: &Weak<ServiceGroup>) -> u16 {
let first_stream_id = self.settings.lock_or_recover().config.first_stream_id;
let mut used_ecm_pids = self.used_ecm_pids.lock_or_recover();
let stream_id = loop {
let offset = self.next_stream_offset.fetch_add(1, Ordering::Relaxed);
let stream_id = first_stream_id.wrapping_add(offset);
if !used_ecm_pids
.values()
.any(|membership| membership.stream_id == stream_id)
{
break stream_id;
}
};
used_ecm_pids.insert(
ecm_pid,
StreamMembership {
stream_id,
group: group.clone(),
},
);
stream_id
}
async fn forward_driver_events(
self: Arc<Self>,
cas: CasRef,
generation: u64,
mut events_rx: mpsc::UnboundedReceiver<DriverEvent>,
) {
while let Some(event) = events_rx.recv().await {
if self.generation.load(Ordering::Relaxed) != generation {
return;
}
match event {
DriverEvent::Ecm {
stream_id,
cp_number,
format,
data,
} => {
let slot = self.stream_sinks.lock_or_recover().get(&stream_id).cloned();
if let Some(slot) = slot {
slot.counters.ecm.fetch_add(1, Ordering::Relaxed);
let _ = slot.sink.send(Event::Ecm {
cas,
stream_id,
cp_number,
format: format.into(),
data,
});
}
}
DriverEvent::StaleEcm { stream_id } => {
if let Some(slot) = self.stream_sinks.lock_or_recover().get(&stream_id) {
slot.counters.ecm_stale.fetch_add(1, Ordering::Relaxed);
}
}
DriverEvent::State(state) => {
let (cas_state, ready_stream_ids) = match state {
DriverState::Connecting => (CasState::Connecting, None),
DriverState::Ready {
section_ts_pkt_flag,
params,
ready_stream_ids,
} => (
CasState::Ready(CasParams::from_wire(section_ts_pkt_flag, params)),
Some(ready_stream_ids),
),
DriverState::Error { code, info } => (CasState::Error { code, info }, None),
DriverState::Disconnected => (CasState::Disconnected, None),
};
match &cas_state {
CasState::Ready(params) => {
self.connects.fetch_add(1, Ordering::Relaxed);
if let Some(min_cp_duration) = params.min_cp_duration {
self.check_min_cp_duration(min_cp_duration);
}
}
CasState::Error { code, info } => {
*self.last_error.lock_or_recover() = Some(CasError {
code: *code,
info: info.clone(),
at: SystemTime::now(),
});
}
CasState::Connecting | CasState::Disconnected => {}
}
*self.state.lock_or_recover() = cas_state.clone();
{
let mut ready = self.ready_streams.lock_or_recover();
if let Some(ids) = ready_stream_ids {
*ready = ids.into_iter().collect();
} else {
ready.clear();
}
}
let sinks: Vec<_> = self
.stream_sinks
.lock_or_recover()
.values()
.map(|slot| slot.sink.clone())
.collect();
for sink in sinks {
let _ = sink.send(Event::State {
cas,
state: cas_state.clone(),
});
}
}
}
}
}
}
#[derive(Clone)]
struct StreamSlot {
sink: broadcast::Sender<Event>,
counters: Arc<StreamCounters>,
}
#[derive(Default)]
struct StreamCounters {
ecm: AtomicU64,
ecm_stale: AtomicU64,
emm: AtomicU64,
}
pub fn register_new_member_io(
cas: CasRef,
entry: &Arc<CasEntry>,
stream_id: u16,
events_tx: &broadcast::Sender<Event>,
nominal_cp_duration: u16,
) {
let slot = StreamSlot {
sink: events_tx.clone(),
counters: Arc::default(),
};
entry
.stream_sinks
.lock_or_recover()
.insert(stream_id, slot.clone());
let (cas_name, emmg) = {
let settings = entry.settings.lock_or_recover();
(settings.config.name.clone(), settings.emmg_channel())
};
if let Some((dispatch, channel_id)) = emmg {
register_emmg_stream(cas, &cas_name, &dispatch, channel_id, stream_id, &slot);
}
add_ecmg_stream(entry, stream_id, nominal_cp_duration);
}
fn register_emmg_stream(
cas: CasRef,
cas_name: &str,
dispatch: &Dispatch,
channel_id: u16,
stream_id: u16,
slot: &StreamSlot,
) {
let (emm_tx, emm_rx) = mpsc::unbounded_channel();
match dispatch.register_stream(channel_id, stream_id, emm_tx) {
Ok(()) => {
spawn_supervised(
"forward-emm-events",
forward_emm_events(cas, emm_rx, slot.clone()),
);
}
Err(error) => {
tracing::warn!(
name = cas_name,
channel_id,
stream_id,
error = %error,
"failed to register EMMG stream"
);
}
}
}
fn add_ecmg_stream(entry: &Arc<CasEntry>, stream_id: u16, nominal_cp_duration: u16) {
let cas_name = entry.settings.lock_or_recover().config.name.clone();
let Some(driver) = entry.driver.lock_or_recover().clone() else {
tracing::warn!(
name = %cas_name,
stream_id,
"ECMG driver is not running, cannot add stream"
);
return;
};
let entry_for_ready = Arc::clone(entry);
let handle = spawn_supervised("add-stream", async move {
match driver.add_stream(stream_id, nominal_cp_duration).await {
Ok(()) => {
entry_for_ready
.ready_streams
.lock_or_recover()
.insert(stream_id);
if let CasState::Ready(params) = &*entry_for_ready.state.lock_or_recover()
&& let Some(min_cp_duration) = params.min_cp_duration
{
warn_if_below_min_cp(
&cas_name,
stream_id,
nominal_cp_duration,
min_cp_duration,
);
}
}
Err(error) => {
tracing::warn!(
name = %cas_name,
stream_id,
error = %error,
"failed to add ECMG stream"
);
}
}
});
entry
.pending_add_stream
.lock_or_recover()
.insert(stream_id, handle);
}
impl CasEntry {
pub fn release_stream(&self, info: &StreamInfo) {
self.stream_sinks.lock_or_recover().remove(&info.stream_id);
self.ready_streams.lock_or_recover().remove(&info.stream_id);
self.used_ecm_pids.lock_or_recover().remove(&info.ecm_pid);
if let Some(handle) = self
.pending_add_stream
.lock_or_recover()
.remove(&info.stream_id)
{
handle.abort();
}
let emmg = self.settings.lock_or_recover().emmg_channel();
if let Some((dispatch, channel_id)) = emmg {
dispatch.unregister_stream(channel_id, info.stream_id);
}
if self.used_ecm_pids.lock_or_recover().is_empty() {
self.shutdown();
} else if let Some(driver) = self.driver.lock_or_recover().clone() {
let _ = driver.remove_stream(info.stream_id);
}
}
}
async fn forward_emm_events(
cas: CasRef,
mut emm_rx: mpsc::UnboundedReceiver<EmmEvent>,
slot: StreamSlot,
) {
while let Some(emm) = emm_rx.recv().await {
slot.counters.emm.fetch_add(1, Ordering::Relaxed);
let _ = slot.sink.send(Event::Emm {
cas,
stream_id: emm.stream_id,
data_id: emm.data_id,
data_type: emm.data_type,
format: emm.format.into(),
data: emm.data,
});
}
}
#[cfg(test)]
mod tests {
use std::net::SocketAddr;
use tokio::net::TcpListener;
use super::*;
use crate::{
EcmgConfig,
ServiceCas,
ServiceConfig,
Simulcrypt,
messages::{
RawSimulcryptMessage,
ecmg::{
ECMG_MSG_CHANNEL_STATUS,
ECMG_MSG_STREAM_SETUP,
ECMG_MSG_STREAM_STATUS,
ECMG_VAR_CHANNEL_ID,
ECMG_VAR_SECTION_TS_PKT_FLAG,
ECMG_VAR_STREAM_ID,
},
},
};
async fn slow_ecmg(listener: TcpListener, channel_id: u16, delay: Duration) {
let Ok((mut socket, _)) = listener.accept().await else {
return;
};
if RawSimulcryptMessage::read_from(&mut socket).await.is_err() {
return;
}
let mut status = RawSimulcryptMessage::new(ECMG_MSG_CHANNEL_STATUS);
status.set_u16(ECMG_VAR_CHANNEL_ID, channel_id);
status.set_u8(ECMG_VAR_SECTION_TS_PKT_FLAG, 0x00);
if status.write_to(&mut socket).await.is_err() {
return;
}
loop {
let Ok(request) = RawSimulcryptMessage::read_from(&mut socket).await else {
return;
};
if request.message_type == ECMG_MSG_STREAM_SETUP
&& let Some(stream_id) = request.get_u16(ECMG_VAR_STREAM_ID)
{
tokio::time::sleep(delay).await;
let mut status = RawSimulcryptMessage::new(ECMG_MSG_STREAM_STATUS);
status.set_u16(ECMG_VAR_CHANNEL_ID, channel_id);
status.set_u16(ECMG_VAR_STREAM_ID, stream_id);
if status.write_to(&mut socket).await.is_err() {
return;
}
}
}
}
#[tokio::test]
async fn dropping_service_before_add_stream_completes_leaves_no_ghost_entries() {
let channel_id = 21;
let super_cas_id = 0x0600_0006;
let delay = Duration::from_millis(300);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
tokio::spawn(slow_ecmg(listener, channel_id, delay));
let mut ecmg_cfg = EcmgConfig::new(channel_id, SocketAddr::from(([127, 0, 0, 1], port)));
ecmg_cfg.reconnect_delay = Duration::from_millis(200);
ecmg_cfg.response_timeout = Duration::from_secs(5);
let simulcrypt = Simulcrypt::new();
let cas_ref = simulcrypt
.add_cas(CasConfig {
id: "cas-ghost".into(),
name: "ghost-cas".into(),
super_cas_id,
first_stream_id: 1,
ecmg: ecmg_cfg,
emmg: None,
})
.unwrap();
let service = simulcrypt
.open_service(&ServiceConfig {
name: "svc-ghost".into(),
crypto_period: Duration::from_millis(150),
cas: vec![ServiceCas {
cas: cas_ref,
ecm_pid: 900,
access_criteria: vec![],
}],
})
.unwrap();
let stream_id = service.streams()[0].stream_id;
drop(service);
tokio::time::sleep(delay * 2).await;
let entry = simulcrypt.entry(cas_ref);
assert!(
!entry.ready_streams.lock_or_recover().contains(&stream_id),
"aborted add_stream task must not resurrect ready_streams after Drop"
);
assert!(
!entry
.pending_add_stream
.lock()
.unwrap()
.contains_key(&stream_id),
"the join handle must be removed once the stream is torn down"
);
assert!(
!entry.used_ecm_pids.lock_or_recover().contains_key(&900),
"ecm_pid must be freed"
);
}
async fn counting_ecmg(
listener: TcpListener,
channel_id: u16,
accept_count: Arc<std::sync::atomic::AtomicUsize>,
) {
loop {
let Ok((mut socket, _)) = listener.accept().await else {
return;
};
accept_count.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if RawSimulcryptMessage::read_from(&mut socket).await.is_err() {
continue;
}
let mut status = RawSimulcryptMessage::new(ECMG_MSG_CHANNEL_STATUS);
status.set_u16(ECMG_VAR_CHANNEL_ID, channel_id);
status.set_u8(ECMG_VAR_SECTION_TS_PKT_FLAG, 0x00);
if status.write_to(&mut socket).await.is_err() {
continue;
}
loop {
let Ok(request) = RawSimulcryptMessage::read_from(&mut socket).await else {
break;
};
if request.message_type == ECMG_MSG_STREAM_SETUP
&& let Some(stream_id) = request.get_u16(ECMG_VAR_STREAM_ID)
{
let mut status = RawSimulcryptMessage::new(ECMG_MSG_STREAM_STATUS);
status.set_u16(ECMG_VAR_CHANNEL_ID, channel_id);
status.set_u16(ECMG_VAR_STREAM_ID, stream_id);
if status.write_to(&mut socket).await.is_err() {
break;
}
}
}
}
}
#[tokio::test]
async fn dropping_last_stream_shuts_down_channel_and_reopening_revives_it() {
let channel_id = 22;
let super_cas_id = 0x0600_0007;
let reconnect_delay = Duration::from_millis(50);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
let accept_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
tokio::spawn(counting_ecmg(
listener,
channel_id,
Arc::clone(&accept_count),
));
let mut ecmg_cfg = EcmgConfig::new(channel_id, SocketAddr::from(([127, 0, 0, 1], port)));
ecmg_cfg.reconnect_delay = reconnect_delay;
ecmg_cfg.response_timeout = Duration::from_secs(5);
let simulcrypt = Simulcrypt::new();
let cas_ref = simulcrypt
.add_cas(CasConfig {
id: "cas-teardown".into(),
name: "teardown-cas".into(),
super_cas_id,
first_stream_id: 1,
ecmg: ecmg_cfg,
emmg: None,
})
.unwrap();
let service = simulcrypt
.open_service(&ServiceConfig {
name: "svc-teardown".into(),
crypto_period: Duration::from_millis(150),
cas: vec![ServiceCas {
cas: cas_ref,
ecm_pid: 1000,
access_criteria: vec![],
}],
})
.unwrap();
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
accept_count.load(std::sync::atomic::Ordering::SeqCst),
1,
"add_cas must connect eagerly"
);
drop(service);
tokio::time::sleep(Duration::from_millis(100)).await;
{
let entry = simulcrypt.entry(cas_ref);
assert!(
entry.driver.lock_or_recover().is_none(),
"driver must be torn down once the channel has no streams left"
);
assert!(matches!(
*entry.state.lock_or_recover(),
CasState::Disconnected
));
}
tokio::time::sleep(reconnect_delay * 3).await;
assert_eq!(
accept_count.load(std::sync::atomic::Ordering::SeqCst),
1,
"a torn-down channel must not keep reconnecting on its own"
);
let service2 = simulcrypt
.open_service(&ServiceConfig {
name: "svc-teardown-2".into(),
crypto_period: Duration::from_millis(150),
cas: vec![ServiceCas {
cas: cas_ref,
ecm_pid: 1001,
access_criteria: vec![],
}],
})
.unwrap();
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
accept_count.load(std::sync::atomic::Ordering::SeqCst),
2,
"reopening a service on the same CasRef must revive the channel with a fresh connection"
);
{
let entry = simulcrypt.entry(cas_ref);
assert!(
entry.driver.lock_or_recover().is_some(),
"driver must be running again after revival"
);
}
drop(service2);
}
}