pub use async_trait::async_trait;
use std::{any::Any, fmt, future::Future, pin::Pin, sync::Arc, time::Duration};
use std::{
collections::BTreeMap,
sync::{
Mutex,
atomic::{AtomicBool, AtomicU64, Ordering},
},
};
use tokio::sync::{mpsc, oneshot, watch};
use tokio::time::{sleep, timeout};
mod pool;
mod queue;
pub use pool::{ManagedResourceProxy, Pool, PoolConfig, QueuedOperation, ResourceLease};
mod proxy;
pub use proxy::{ManagedClientBackend, SupervisedResourceProxy};
pub use queue::{FifoQueue, OperationQueue, PriorityQueue, QueueError};
#[async_trait]
pub trait Lifecycle: Send + Sync + 'static {
type Resource: Send + Sync + 'static;
type Error: std::error::Error + Send + Sync + 'static;
async fn create(&self) -> Result<Self::Resource, Self::Error>;
async fn connect(&self, resource: &Self::Resource) -> Result<(), Self::Error>;
async fn on_resource_created(&self, _resource: &Self::Resource) -> Result<(), Self::Error> {
Ok(())
}
async fn setup(&self, _resource: &Self::Resource) -> Result<(), Self::Error> {
Ok(())
}
async fn cleanup(&self, _resource: &Self::Resource) -> Result<(), Self::Error> {
Ok(())
}
async fn disconnect(&self, resource: &Self::Resource) -> Result<(), Self::Error>;
async fn destroy(&self, _resource: &Self::Resource) -> Result<(), Self::Error> {
Ok(())
}
fn watch_disconnect(
&self,
_resource: Arc<Self::Resource>,
) -> Option<LifecycleFuture<Self::Error>> {
None
}
fn is_connected(&self, _resource: &Self::Resource) -> bool {
true
}
async fn wait_connected(&self, _resource: &Self::Resource) {
std::future::pending::<()>().await
}
fn is_terminal(&self, _error: &Self::Error) -> bool {
false
}
fn is_expected(&self, _error: &Self::Error) -> bool {
false
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ResourceState {
Connecting,
Connected,
Recovering,
Disconnected,
Stopping,
Stopped,
}
#[derive(Clone, Debug)]
pub struct Config {
pub resource_name: String,
pub create_timeout: Option<Duration>,
pub created_timeout: Option<Duration>,
pub connect_timeout: Duration,
pub setup_timeout: Duration,
pub cleanup_timeout: Duration,
pub disconnect_timeout: Duration,
pub destroy_timeout: Option<Duration>,
pub retry_delay: Duration,
pub max_retry_delay: Duration,
}
impl Default for Config {
fn default() -> Self {
Self {
resource_name: "Resource".into(),
create_timeout: None,
created_timeout: None,
connect_timeout: Duration::from_secs(20),
setup_timeout: Duration::from_secs(10),
cleanup_timeout: Duration::from_secs(10),
disconnect_timeout: Duration::from_secs(10),
destroy_timeout: None,
retry_delay: Duration::from_secs(5),
max_retry_delay: Duration::from_secs(300),
}
}
}
#[derive(Debug)]
pub enum Error<E> {
Stopped,
Timeout,
Operation(E),
Queue(QueueError),
}
impl<E: fmt::Display> fmt::Display for Error<E> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Stopped => f.write_str("resource supervisor stopped"),
Self::Timeout => f.write_str("operation retry deadline elapsed"),
Self::Operation(e) => e.fmt(f),
Self::Queue(e) => e.fmt(f),
}
}
}
impl<E: std::error::Error + 'static> std::error::Error for Error<E> {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Self::Operation(e) => Some(e),
Self::Stopped | Self::Timeout => None,
Self::Queue(e) => Some(e),
}
}
}
#[derive(Clone, Copy, Debug)]
pub struct RetryPolicy {
pub max_attempts: std::num::NonZeroUsize,
pub timeout: Duration,
}
pub struct ResourceHandle<R> {
resource: Arc<R>,
generation: u64,
}
impl<R> Clone for ResourceHandle<R> {
fn clone(&self) -> Self {
Self {
resource: self.resource.clone(),
generation: self.generation,
}
}
}
impl<R> ResourceHandle<R> {
pub fn resource(&self) -> &Arc<R> {
&self.resource
}
pub fn generation(&self) -> u64 {
self.generation
}
}
pub struct ResourceStatus<R> {
pub state: ResourceState,
pub handle: Option<ResourceHandle<R>>,
}
type Snapshot<R> = ResourceStatus<R>;
impl<R> Clone for Snapshot<R> {
fn clone(&self) -> Self {
Self {
state: self.state,
handle: self.handle.clone(),
}
}
}
struct Owner {
stop: watch::Sender<bool>,
started: AtomicBool,
}
impl Drop for Owner {
fn drop(&mut self) {
self.stop.send_replace(true);
}
}
struct Recovery {
generation: u64,
terminal: bool,
completed: Option<oneshot::Sender<()>>,
}
pub type LifecycleFuture<E> = Pin<Box<dyn Future<Output = Result<(), E>> + Send>>;
pub type CreatedHook<R, E> = Arc<dyn Fn(Arc<R>) -> LifecycleFuture<E> + Send + Sync>;
pub type AttributeSetter<R> = Arc<dyn Fn(&R) + Send + Sync>;
struct Attributes<R, E> {
setters: BTreeMap<String, AttributeSetter<R>>,
last: Option<Arc<R>>,
values: BTreeMap<String, Arc<dyn Any + Send + Sync>>,
on_created: Option<CreatedHook<R, E>>,
after_created: Option<CreatedHook<R, E>>,
}
pub struct Supervisor<L: Lifecycle> {
lifecycle: Arc<L>,
owner: Arc<Owner>,
snapshot: watch::Receiver<Snapshot<L::Resource>>,
recovery: mpsc::UnboundedSender<Recovery>,
attributes: Arc<Mutex<Attributes<L::Resource, L::Error>>>,
config: Config,
sender: watch::Sender<Snapshot<L::Resource>>,
driver: Arc<tokio::sync::Mutex<Option<mpsc::UnboundedReceiver<Recovery>>>>,
generations: Arc<AtomicU64>,
}
impl<L: Lifecycle> Clone for Supervisor<L> {
fn clone(&self) -> Self {
Self {
lifecycle: self.lifecycle.clone(),
owner: self.owner.clone(),
snapshot: self.snapshot.clone(),
recovery: self.recovery.clone(),
attributes: self.attributes.clone(),
config: self.config.clone(),
sender: self.sender.clone(),
driver: self.driver.clone(),
generations: self.generations.clone(),
}
}
}
impl<L: Lifecycle> Supervisor<L> {
pub fn start(lifecycle: L, config: Config) -> Self {
let supervisor = Self::new(lifecycle, config);
supervisor.begin();
supervisor
}
pub fn new(lifecycle: L, config: Config) -> Self {
let lifecycle = Arc::new(lifecycle);
let (stop, stopping) = watch::channel(false);
let (sender, snapshot) = watch::channel(Snapshot {
state: ResourceState::Disconnected,
handle: None,
});
let (recovery, requests) = mpsc::unbounded_channel();
let attributes = Arc::new(Mutex::new(Attributes {
setters: BTreeMap::new(),
last: None,
values: BTreeMap::new(),
on_created: None,
after_created: None,
}));
drop(stopping);
Self {
lifecycle,
owner: Arc::new(Owner {
stop,
started: AtomicBool::new(false),
}),
snapshot,
recovery,
attributes,
config,
sender,
driver: Arc::new(tokio::sync::Mutex::new(Some(requests))),
generations: Arc::new(AtomicU64::new(0)),
}
}
pub fn begin(&self) {
if self.sender.send_if_modified(|status| {
if status.state == ResourceState::Stopped {
status.state = ResourceState::Connecting;
true
} else {
false
}
}) {
self.owner.stop.send_replace(false);
self.owner.started.store(true, Ordering::Release);
self.launch();
return;
}
if !*self.owner.stop.borrow() && !self.owner.started.swap(true, Ordering::AcqRel) {
publish(&self.sender, ResourceState::Connecting, None);
self.launch();
}
}
fn launch(&self) {
let driver = self.driver.clone();
let lifecycle = self.lifecycle.clone();
let config = self.config.clone();
let stopping = self.owner.stop.subscribe();
let sender = self.sender.clone();
let attributes = self.attributes.clone();
let generations = self.generations.clone();
tokio::spawn(async move {
let mut guard = driver.lock().await;
let requests = guard.take().expect("only one lifecycle worker");
*guard = Some(
maintain(
lifecycle,
config,
stopping,
sender,
requests,
attributes,
generations,
)
.await,
);
});
}
pub async fn restart(&self) -> bool {
if self.state() != ResourceState::Stopped {
return false;
}
let guard = self.driver.lock().await;
if self.state() != ResourceState::Stopped {
return false;
}
self.owner.stop.send_replace(false);
self.owner.started.store(true, Ordering::Release);
publish(&self.sender, ResourceState::Connecting, None);
drop(guard);
self.launch();
true
}
pub fn subscribe(&self) -> watch::Receiver<ResourceStatus<L::Resource>> {
self.snapshot.clone()
}
pub fn current(&self) -> Option<ResourceHandle<L::Resource>> {
self.snapshot.borrow().handle.clone()
}
pub fn with_latest<T>(&self, read: impl FnOnce(&L::Resource) -> T) -> Option<T> {
self.attributes.lock().unwrap().last.as_deref().map(read)
}
pub fn set_attribute(&self, name: impl Into<String>, setter: AttributeSetter<L::Resource>) {
let mut attributes = self.attributes.lock().unwrap();
if let Some(resource) = &attributes.last {
setter(resource);
}
attributes.setters.insert(name.into(), setter);
}
pub fn set_value<T: Clone + Send + Sync + 'static>(
&self,
name: impl Into<String>,
value: T,
apply: impl Fn(&L::Resource, &T) + Send + Sync + 'static,
) {
let name = name.into();
let value = Arc::new(value);
let stored = value.clone();
let setter: AttributeSetter<L::Resource> =
Arc::new(move |resource| apply(resource, &value));
let mut attributes = self.attributes.lock().unwrap();
attributes.values.insert(name.clone(), stored);
if let Some(resource) = &attributes.last {
setter(resource);
}
attributes.setters.insert(name, setter);
}
pub fn value<T: Clone + Send + Sync + 'static>(&self, name: &str) -> Option<T> {
self.attributes
.lock()
.unwrap()
.values
.get(name)?
.downcast_ref::<T>()
.cloned()
}
pub fn set_on_resource_created(&self, hook: CreatedHook<L::Resource, L::Error>) {
self.attributes.lock().unwrap().on_created = Some(hook);
}
pub fn is_connected(&self) -> bool {
self.current()
.is_some_and(|handle| self.lifecycle.is_connected(&handle.resource))
}
pub fn stopped(&self) -> watch::Receiver<bool> {
self.owner.stop.subscribe()
}
pub fn state(&self) -> ResourceState {
self.snapshot.borrow().state
}
pub async fn acquire(&self) -> Result<ResourceHandle<L::Resource>, Error<L::Error>> {
self.begin();
self.acquire_running().await
}
pub(crate) async fn acquire_running(
&self,
) -> Result<ResourceHandle<L::Resource>, Error<L::Error>> {
let mut snapshot = self.snapshot.clone();
loop {
if *self.owner.stop.borrow() || snapshot.has_changed().is_err() {
return Err(Error::Stopped);
}
if let Some(handle) = snapshot.borrow_and_update().handle.clone() {
return Ok(handle);
}
snapshot.changed().await.map_err(|_| Error::Stopped)?;
}
}
pub fn request_recovery(&self, handle: &ResourceHandle<L::Resource>, terminal: bool) {
let _ = self.submit_recovery(handle, terminal, None);
}
pub async fn recover(
&self,
handle: &ResourceHandle<L::Resource>,
terminal: bool,
) -> Result<(), Error<L::Error>> {
let (sender, receiver) = oneshot::channel();
if self.submit_recovery(handle, terminal, Some(sender)) {
receiver.await.map_err(|_| Error::Stopped)?;
}
Ok(())
}
pub async fn recover_current(&self, terminal: bool) -> Result<(), Error<L::Error>> {
if let Some(handle) = self.current() {
self.recover(&handle, terminal).await?;
}
Ok(())
}
fn submit_recovery(
&self,
handle: &ResourceHandle<L::Resource>,
terminal: bool,
completed: Option<oneshot::Sender<()>>,
) -> bool {
let mut matched = self.sender.send_if_modified(|snapshot| {
if snapshot
.handle
.as_ref()
.is_some_and(|current| Arc::ptr_eq(¤t.resource, &handle.resource))
{
snapshot.handle = None;
snapshot.state = ResourceState::Recovering;
true
} else {
false
}
});
if !matched && self.state() == ResourceState::Recovering {
matched = self
.attributes
.lock()
.unwrap()
.last
.as_ref()
.is_some_and(|resource| Arc::ptr_eq(resource, &handle.resource));
}
if matched {
let _ = self.recovery.send(Recovery {
generation: handle.generation,
terminal,
completed,
});
}
matched
}
pub async fn execute<F, Fut, T>(&self, operation: F) -> Result<T, Error<L::Error>>
where
F: FnOnce(Arc<L::Resource>) -> Fut,
Fut: Future<Output = Result<T, L::Error>>,
{
self.begin();
self.execute_running(operation).await
}
pub(crate) async fn execute_running<F, Fut, T>(
&self,
operation: F,
) -> Result<T, Error<L::Error>>
where
F: FnOnce(Arc<L::Resource>) -> Fut,
Fut: Future<Output = Result<T, L::Error>>,
{
let handle = self.acquire_running().await?;
let mut stopping = self.owner.stop.subscribe();
let result = tokio::select! {
biased;
_ = shutdown(&mut stopping) => return Err(Error::Stopped),
result = operation(handle.resource.clone()) => result,
};
match result {
Ok(value) => Ok(value),
Err(error) => {
let _ = self
.recover(&handle, self.lifecycle.is_terminal(&error))
.await;
Err(Error::Operation(error))
}
}
}
pub async fn execute_with_retry<F, Fut, T, P>(
&self,
policy: RetryPolicy,
operation: F,
retry_if: P,
) -> Result<T, Error<L::Error>>
where
F: FnMut(Arc<L::Resource>) -> Fut,
Fut: Future<Output = Result<T, L::Error>>,
P: FnMut(&L::Error) -> bool,
{
if policy.timeout.is_zero() {
return Err(Error::Timeout);
}
self.begin();
self.execute_with_retry_running(policy, operation, retry_if)
.await
}
pub(crate) async fn execute_with_retry_running<F, Fut, T, P>(
&self,
policy: RetryPolicy,
mut operation: F,
mut retry_if: P,
) -> Result<T, Error<L::Error>>
where
F: FnMut(Arc<L::Resource>) -> Fut,
Fut: Future<Output = Result<T, L::Error>>,
P: FnMut(&L::Error) -> bool,
{
if policy.timeout.is_zero() {
return Err(Error::Timeout);
}
let mut stopping = self.owner.stop.subscribe();
let retry_stopping = stopping.clone();
let attempts = async {
for attempt in 1..=policy.max_attempts.get() {
if *retry_stopping.borrow() {
return Err(Error::Stopped);
}
match self.execute_running(&mut operation).await {
Err(Error::Operation(error))
if attempt < policy.max_attempts.get() && retry_if(&error) => {}
result => return result,
}
}
unreachable!("a nonzero final attempt always returns")
};
tokio::select! {
biased;
_ = shutdown(&mut stopping) => Err(Error::Stopped),
result = timeout(policy.timeout, attempts) => result.unwrap_or(Err(Error::Timeout)),
}
}
pub async fn stop(&self) {
self.request_stop();
let mut snapshot = self.snapshot.clone();
loop {
if snapshot.borrow_and_update().state == ResourceState::Stopped {
drop(self.driver.lock().await);
return;
}
if snapshot.changed().await.is_err() {
return;
}
}
}
pub(crate) fn request_stop(&self) {
self.owner.stop.send_replace(true);
if !self.owner.started.load(Ordering::Acquire) {
publish(&self.sender, ResourceState::Stopped, None);
} else {
self.sender.send_if_modified(|status| {
if status.state == ResourceState::Stopped {
false
} else {
status.state = ResourceState::Stopping;
status.handle = None;
true
}
});
}
}
}
async fn shutdown(receiver: &mut watch::Receiver<bool>) {
loop {
if *receiver.borrow_and_update() {
return;
}
if receiver.changed().await.is_err() {
return;
}
}
}
fn publish<R>(
sender: &watch::Sender<Snapshot<R>>,
state: ResourceState,
handle: Option<ResourceHandle<R>>,
) {
sender.send_replace(Snapshot { state, handle });
}
fn log_failure<L: Lifecycle>(lifecycle: &L, config: &Config, name: &str, error: &L::Error) {
if lifecycle.is_expected(error) {
tracing::warn!(resource = config.resource_name, hook = name, %error);
tracing::debug!(resource = config.resource_name, hook = name, ?error);
} else {
tracing::error!(resource = config.resource_name, hook = name, %error);
}
}
fn log_late_failure<L: Lifecycle>(lifecycle: &L, config: &Config, name: &str, error: &L::Error) {
if lifecycle.is_expected(error) {
tracing::debug!(
resource = config.resource_name,
hook = name,
?error,
"abandoned hook failed"
);
} else {
log_failure(lifecycle, config, name, error);
}
}
async fn bounded<L: Lifecycle>(
lifecycle: Arc<L>,
config: Config,
name: &'static str,
duration: Option<Duration>,
future: impl Future<Output = Result<(), L::Error>> + Send + 'static,
) -> Result<(), bool> {
let mut task = tokio::spawn(future);
match deadline(duration, &mut task).await {
Ok(Ok(Ok(()))) => Ok(()),
Ok(Ok(Err(error))) => {
log_failure(lifecycle.as_ref(), &config, name, &error);
Err(lifecycle.is_terminal(&error))
}
Ok(Err(error)) => {
tracing::error!(hook = name, %error, "hook task failed");
Err(false)
}
Err(_) => {
tracing::warn!(
resource = config.resource_name,
hook = name,
"hook abandoned after timeout"
);
tokio::spawn(async move {
match task.await {
Ok(Err(error)) => log_late_failure(lifecycle.as_ref(), &config, name, &error),
Ok(Ok(())) => tracing::warn!(
resource = config.resource_name,
hook = name,
"abandoned hook completed after timeout"
),
Err(error) => {
tracing::error!(hook = name, %error, "abandoned hook task failed")
}
}
});
Err(false)
}
}
}
async fn deadline<T>(
duration: Option<Duration>,
future: impl Future<Output = T>,
) -> Result<T, tokio::time::error::Elapsed> {
match duration {
Some(duration) => timeout(duration, future).await,
None => Ok(future.await),
}
}
async fn destroy<L: Lifecycle>(lifecycle: Arc<L>, config: Config, resource: Arc<L::Resource>) {
let manager = lifecycle.clone();
let _ = bounded(
lifecycle,
config.clone(),
"destroy",
config.destroy_timeout,
async move { manager.destroy(&resource).await },
)
.await;
}
async fn close<L: Lifecycle>(
lifecycle: Arc<L>,
config: Config,
resource: Arc<L::Resource>,
terminal: bool,
) {
let manager = lifecycle.clone();
let value = resource.clone();
let _ = bounded(
lifecycle.clone(),
config.clone(),
"cleanup",
Some(config.cleanup_timeout),
async move { manager.cleanup(&value).await },
)
.await;
if !terminal {
let manager = lifecycle.clone();
let value = resource.clone();
let mut task = tokio::spawn(async move { manager.disconnect(&value).await });
match timeout(config.disconnect_timeout, &mut task).await {
Ok(Ok(Err(error))) => log_failure(lifecycle.as_ref(), &config, "disconnect", &error),
Ok(Err(error)) => tracing::error!(%error, "disconnect task failed"),
Ok(Ok(Ok(()))) => {}
Err(_) => {
tracing::warn!(
resource = config.resource_name,
"disconnect abandoned after timeout"
);
tokio::spawn(async move {
if let Ok(Err(error)) = task.await {
log_late_failure(lifecycle.as_ref(), &config, "disconnect", &error);
}
destroy(lifecycle, config, resource).await;
});
return;
}
}
}
destroy(lifecycle, config, resource).await;
}
async fn connect<L: Lifecycle>(
lifecycle: Arc<L>,
config: Config,
resource: Arc<L::Resource>,
stopping: &mut watch::Receiver<bool>,
) -> Result<(), bool> {
let manager = lifecycle.clone();
let value = resource.clone();
let mut task = tokio::spawn(async move { manager.connect(&value).await });
tokio::select! {
biased;
_ = shutdown(stopping) => { task.abort(); let _ = task.await; Err(false) }
result = timeout(config.connect_timeout, &mut task) => match result {
Ok(Ok(Ok(()))) => Ok(()),
Ok(Ok(Err(error))) => {
log_failure(lifecycle.as_ref(), &config, "connect", &error);
Err(lifecycle.is_terminal(&error))
}
Ok(Err(error)) => { tracing::error!(%error, "connect task failed"); Err(false) }
Err(_) => {
tracing::warn!(resource = config.resource_name, "connect abandoned after timeout");
tokio::spawn(async move {
match task.await {
Ok(Ok(())) => close(lifecycle, config, resource, false).await,
Ok(Err(error)) => log_late_failure(lifecycle.as_ref(), &config, "late connect", &error),
Err(error) => tracing::error!(%error, "late connect task failed"),
}
});
Err(false)
}
}
}
}
async fn maintain<L: Lifecycle>(
lifecycle: Arc<L>,
config: Config,
mut stopping: watch::Receiver<bool>,
sender: watch::Sender<Snapshot<L::Resource>>,
mut requests: mpsc::UnboundedReceiver<Recovery>,
attributes: Arc<Mutex<Attributes<L::Resource, L::Error>>>,
generations: Arc<AtomicU64>,
) -> mpsc::UnboundedReceiver<Recovery> {
let mut delay = config.retry_delay;
loop {
while let Ok(request) = requests.try_recv() {
if let Some(completed) = request.completed {
let _ = completed.send(());
}
}
if *stopping.borrow() {
break;
}
publish(&sender, ResourceState::Connecting, None);
let manager = lifecycle.clone();
let mut creating = tokio::spawn(async move { manager.create().await });
let created = tokio::select! {
biased;
_ = shutdown(&mut stopping) => { creating.abort(); let _ = creating.await; break },
result = deadline(config.create_timeout, &mut creating) => result,
};
match created {
Ok(Ok(Ok(resource))) => {
let resource = Arc::new(resource);
let (on_created, after_created) = {
let mut attributes = attributes.lock().unwrap();
attributes.last = Some(resource.clone());
(
attributes.on_created.clone(),
attributes.after_created.clone(),
)
};
let manager = lifecycle.clone();
let value = resource.clone();
let settings = attributes.clone();
let created = bounded(
lifecycle.clone(),
config.clone(),
"on_resource_created",
config.created_timeout,
async move {
manager.on_resource_created(&value).await?;
if let Some(hook) = on_created {
hook(value.clone()).await?;
}
{
let settings = settings.lock().unwrap();
for setter in settings.setters.values() {
setter(&value);
}
}
if let Some(hook) = after_created {
hook(value).await?;
}
Ok(())
},
);
let created = tokio::select! { biased; _ = shutdown(&mut stopping) => Err(false), result = created => result };
let connected = if created.is_ok() {
connect(
lifecycle.clone(),
config.clone(),
resource.clone(),
&mut stopping,
)
.await
} else {
created
};
let manager = lifecycle.clone();
let value = resource.clone();
let setup = if connected.is_ok() {
tokio::select! {
biased;
_ = shutdown(&mut stopping) => Err(false),
ready = bounded(lifecycle.clone(), config.clone(), "setup", Some(config.setup_timeout),
async move { manager.setup(&value).await }) => ready,
}
} else {
connected
};
let terminal = setup.err().unwrap_or(false);
if setup.is_ok() {
let generation = generations.fetch_add(1, Ordering::AcqRel) + 1;
delay = config.retry_delay;
tracing::info!(resource = config.resource_name, "connected");
publish(
&sender,
ResourceState::Connected,
Some(ResourceHandle {
resource: resource.clone(),
generation,
}),
);
let mut watchdog = lifecycle
.watch_disconnect(resource.clone())
.map(tokio::spawn);
let mut watchdog_done = false;
let mut completed = None;
let terminal = loop {
tokio::select! {
biased;
_ = shutdown(&mut stopping) => break false,
request = requests.recv() => {
match request {
Some(request) if request.generation == generation => { completed = request.completed; break request.terminal },
Some(request) => { if let Some(completed) = request.completed { let _ = completed.send(()); } continue },
None => break false,
}
}
result = async { match watchdog.as_mut() { Some(task) => task.await, None => std::future::pending().await } } => {
watchdog_done = true;
break match result { Ok(Err(error)) => { log_failure(lifecycle.as_ref(), &config, "watch_disconnect", &error); lifecycle.is_terminal(&error) }, _ => false };
}
}
};
if !watchdog_done && let Some(watchdog) = watchdog {
watchdog.abort();
let _ = watchdog.await;
}
let state = if *stopping.borrow() {
ResourceState::Stopping
} else {
ResourceState::Recovering
};
publish(&sender, state, None);
close(lifecycle.clone(), config.clone(), resource, terminal).await;
if let Some(completed) = completed {
let _ = completed.send(());
}
continue;
}
publish(&sender, ResourceState::Disconnected, None);
close(lifecycle.clone(), config.clone(), resource, terminal).await;
}
Ok(Ok(Err(error))) => log_failure(lifecycle.as_ref(), &config, "create", &error),
Ok(Err(error)) => tracing::error!(%error, "creation task failed"),
Err(_) => {
tracing::warn!("resource creation timed out");
let manager = lifecycle.clone();
let settings = config.clone();
tokio::spawn(async move {
if let Ok(Ok(resource)) = creating.await {
close(manager, settings, Arc::new(resource), false).await;
}
});
}
}
publish(&sender, ResourceState::Disconnected, None);
tracing::info!(resource = config.resource_name, retry_delay = ?delay, "retrying connection");
tokio::select! { biased; _ = shutdown(&mut stopping) => break, _ = sleep(delay) => {} }
delay = delay.saturating_mul(2).min(config.max_retry_delay);
}
while let Ok(request) = requests.try_recv() {
if let Some(completed) = request.completed {
let _ = completed.send(());
}
}
publish(&sender, ResourceState::Stopped, None);
requests
}