use crate::{
AttributeSetter, CreatedHook, Error, FifoQueue, Lifecycle, OperationQueue, PriorityQueue,
ResourceHandle, ResourceState, RetryPolicy, Supervisor,
};
use std::{
any::Any,
collections::BTreeMap,
future::Future,
pin::Pin,
sync::{
Arc, Mutex,
atomic::{AtomicBool, AtomicUsize, Ordering},
},
time::Duration,
};
use tokio::{
sync::{Notify, oneshot, watch},
task::JoinSet,
time::{Instant, sleep},
};
#[derive(Clone, Debug)]
pub struct PoolConfig {
pub min_size: usize,
pub max_size: usize,
pub idle_timeout: Duration,
pub priority_queue: bool,
}
impl Default for PoolConfig {
fn default() -> Self {
Self {
min_size: 1,
max_size: 1,
idle_timeout: Duration::from_secs(60),
priority_queue: false,
}
}
}
struct Entry<L: Lifecycle> {
supervisor: Supervisor<L>,
leased: AtomicBool,
idle_since: Mutex<Instant>,
relay: Mutex<Option<tokio::task::JoinHandle<()>>>,
}
type Job<L> = Box<dyn FnOnce(ResourceLease<L>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send>;
pub struct QueuedOperation<L: Lifecycle> {
priority: i32,
sequence: u64,
job: Job<L>,
cancelled: Arc<AtomicBool>,
}
impl<L: Lifecycle> QueuedOperation<L> {
pub fn priority(&self) -> i32 {
self.priority
}
pub fn sequence(&self) -> u64 {
self.sequence
}
pub fn is_cancelled(&self) -> bool {
self.cancelled.load(Ordering::Acquire)
}
}
type Operation<L> = QueuedOperation<L>;
impl<L: Lifecycle> PartialEq for Operation<L> {
fn eq(&self, other: &Self) -> bool {
(self.priority, self.sequence) == (other.priority, other.sequence)
}
}
impl<L: Lifecycle> Eq for Operation<L> {}
impl<L: Lifecycle> PartialOrd for Operation<L> {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl<L: Lifecycle> Ord for Operation<L> {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
(other.priority, other.sequence).cmp(&(self.priority, self.sequence))
}
}
struct Contents<L: Lifecycle> {
entries: Vec<Arc<Entry<L>>>,
queue: Box<dyn OperationQueue<QueuedOperation<L>>>,
sequence: u64,
attributes: BTreeMap<String, AttributeSetter<L::Resource>>,
values: BTreeMap<String, Arc<dyn Any + Send + Sync>>,
retiring: usize,
}
struct Shared<L: Lifecycle> {
config: PoolConfig,
factory: Box<dyn Fn() -> Supervisor<L> + Send + Sync>,
contents: Mutex<Contents<L>>,
operations: Mutex<JoinSet<()>>,
fast_admission: bool,
changed: watch::Sender<u64>,
dispatcher: Arc<Notify>,
stopping: watch::Sender<bool>,
done: watch::Sender<bool>,
waiters: AtomicUsize,
last_resource: Mutex<Option<Arc<L::Resource>>>,
on_created: Mutex<Option<CreatedHook<L::Resource, L::Error>>>,
}
impl<L: Lifecycle> Shared<L> {
fn apply_attribute(
&self,
contents: &Contents<L>,
name: &str,
setter: &AttributeSetter<L::Resource>,
) {
let last_resource = self.last_resource.lock().unwrap().clone();
let mut last_updated = false;
for entry in &contents.entries {
let mut attributes = entry.supervisor.attributes.lock().unwrap();
if let Some(resource) = &attributes.last {
setter(resource);
last_updated |= last_resource
.as_ref()
.is_some_and(|last| Arc::ptr_eq(last, resource));
}
attributes.setters.insert(name.to_owned(), setter.clone());
}
if !last_updated && let Some(resource) = last_resource {
setter(&resource);
}
}
fn notify(&self) {
self.dispatcher.notify_one();
self.changed
.send_modify(|revision| *revision = revision.wrapping_add(1));
}
fn grow(self: &Arc<Self>, contents: &mut Contents<L>) {
let supervisor = (self.factory)();
for (name, setter) in &contents.attributes {
supervisor.set_attribute(name.clone(), setter.clone());
}
let weak = Arc::downgrade(self);
supervisor.attributes.lock().unwrap().after_created = Some(Arc::new(move |resource| {
let shared = weak.upgrade();
Box::pin(async move {
if let Some(shared) = shared {
*shared.last_resource.lock().unwrap() = Some(resource.clone());
let hook = shared.on_created.lock().unwrap().clone();
if let Some(hook) = hook {
hook(resource).await?;
}
}
Ok(())
})
}));
supervisor.begin();
let entry = Arc::new(Entry {
supervisor,
leased: AtomicBool::new(false),
idle_since: Mutex::new(Instant::now()),
relay: Mutex::new(None),
});
let state = entry.supervisor.subscribe();
let changed = self.changed.clone();
let dispatcher = self.dispatcher.clone();
let relay = tokio::spawn(relay_state(state, changed, dispatcher));
*entry.relay.lock().unwrap() = Some(relay);
contents.entries.push(entry);
self.notify();
}
}
struct PoolOwner {
stopping: watch::Sender<bool>,
started: AtomicBool,
}
async fn relay_state<R: Send + Sync + 'static>(
mut state: watch::Receiver<crate::ResourceStatus<R>>,
changed: watch::Sender<u64>,
dispatcher: Arc<Notify>,
) {
loop {
let stopped = state.borrow_and_update().state == ResourceState::Stopped;
changed.send_modify(|revision| *revision = revision.wrapping_add(1));
dispatcher.notify_one();
if stopped || state.changed().await.is_err() {
break;
}
}
}
impl Drop for PoolOwner {
fn drop(&mut self) {
self.stopping.send_replace(true);
}
}
pub struct Pool<L: Lifecycle> {
shared: Arc<Shared<L>>,
owner: Arc<PoolOwner>,
}
impl<L: Lifecycle> Clone for Pool<L> {
fn clone(&self) -> Self {
Self {
shared: self.shared.clone(),
owner: self.owner.clone(),
}
}
}
impl<L: Lifecycle> Pool<L> {
pub fn start(
factory: impl Fn() -> Supervisor<L> + Send + Sync + 'static,
config: PoolConfig,
) -> Self {
let pool = Self::new(factory, config);
pool.begin();
pool
}
pub fn new(
factory: impl Fn() -> Supervisor<L> + Send + Sync + 'static,
config: PoolConfig,
) -> Self {
if config.priority_queue {
Self::with_queue(factory, config, PriorityQueue::default, true)
} else {
Self::with_queue(factory, config, FifoQueue::default, true)
}
}
pub fn start_with_queue_factory<Q>(
factory: impl Fn() -> Supervisor<L> + Send + Sync + 'static,
config: PoolConfig,
queue_factory: impl FnOnce() -> Q,
) -> Self
where
Q: OperationQueue<QueuedOperation<L>> + 'static,
{
let pool = Self::new_with_queue_factory(factory, config, queue_factory);
pool.begin();
pool
}
pub fn new_with_queue_factory<Q>(
factory: impl Fn() -> Supervisor<L> + Send + Sync + 'static,
config: PoolConfig,
queue_factory: impl FnOnce() -> Q,
) -> Self
where
Q: OperationQueue<QueuedOperation<L>> + 'static,
{
Self::with_queue(factory, config, queue_factory, false)
}
fn with_queue<Q>(
factory: impl Fn() -> Supervisor<L> + Send + Sync + 'static,
config: PoolConfig,
queue_factory: impl FnOnce() -> Q,
fast_admission: bool,
) -> Self
where
Q: OperationQueue<QueuedOperation<L>> + 'static,
{
assert!(
config.max_size > 0 && config.min_size <= config.max_size,
"invalid pool capacity"
);
assert!(
!config.idle_timeout.is_zero(),
"idle timeout must be positive"
);
let queue = queue_factory();
assert!(queue.is_empty(), "queue factory must return an empty queue");
let (changed, _) = watch::channel(0);
let (stopping, _) = watch::channel(false);
let (done, _) = watch::channel(false);
let shared = Arc::new(Shared {
config,
factory: Box::new(factory),
contents: Mutex::new(Contents {
entries: Vec::new(),
queue: Box::new(queue),
sequence: 0,
attributes: BTreeMap::new(),
values: BTreeMap::new(),
retiring: 0,
}),
operations: Mutex::new(JoinSet::new()),
fast_admission,
changed,
dispatcher: Arc::new(Notify::new()),
stopping,
done,
waiters: AtomicUsize::new(0),
last_resource: Mutex::new(None),
on_created: Mutex::new(None),
});
Self {
owner: Arc::new(PoolOwner {
stopping: shared.stopping.clone(),
started: AtomicBool::new(false),
}),
shared,
}
}
pub fn begin(&self) {
if *self.shared.stopping.borrow() || self.owner.started.swap(true, Ordering::AcqRel) {
return;
}
{
let mut contents = self.shared.contents.lock().unwrap();
while contents.entries.len() < self.shared.config.min_size {
self.shared.grow(&mut contents);
}
}
tokio::spawn(run(self.shared.clone()));
}
pub fn size(&self) -> usize {
self.shared.contents.lock().unwrap().entries.len()
}
pub fn supervisors(&self) -> Vec<Supervisor<L>> {
self.shared
.contents
.lock()
.unwrap()
.entries
.iter()
.map(|entry| entry.supervisor.clone())
.collect()
}
pub fn resources(&self) -> Vec<ResourceHandle<L::Resource>> {
self.supervisors()
.iter()
.filter_map(Supervisor::current)
.collect()
}
pub fn with_latest<T>(&self, read: impl FnOnce(&L::Resource) -> T) -> Option<T> {
let resource = self
.resources()
.first()
.map(|handle| handle.resource().clone())
.or_else(|| self.shared.last_resource.lock().unwrap().clone());
resource.as_deref().map(read)
}
pub fn subscribe(&self) -> watch::Receiver<u64> {
self.shared.changed.subscribe()
}
pub fn state(&self) -> ResourceState {
if *self.shared.stopping.borrow() {
return if *self.shared.done.borrow() {
ResourceState::Stopped
} else {
ResourceState::Stopping
};
}
let states: Vec<_> = self.supervisors().iter().map(Supervisor::state).collect();
for candidate in [
ResourceState::Connected,
ResourceState::Connecting,
ResourceState::Recovering,
] {
if states.contains(&candidate) {
return candidate;
}
}
ResourceState::Disconnected
}
pub async fn connected(&self) -> Result<(), Error<L::Error>> {
let mut changed = self.subscribe();
let mut stopping = self.shared.stopping.subscribe();
loop {
if *stopping.borrow() {
return Err(Error::Stopped);
}
if self.is_connected() {
return Ok(());
}
let mut connections = JoinSet::new();
for supervisor in self.supervisors() {
if let Some(resource) = supervisor.current() {
let lifecycle = supervisor.lifecycle.clone();
connections.spawn(async move {
lifecycle.wait_connected(&resource.resource).await;
});
}
}
if self.is_connected() {
connections.abort_all();
while connections.join_next().await.is_some() {}
return Ok(());
}
tokio::select! { _ = crate::shutdown(&mut stopping) => {}, _ = changed.changed() => {}, _ = connections.join_next(), if !connections.is_empty() => {} }
connections.abort_all();
while connections.join_next().await.is_some() {}
}
}
pub fn is_connected(&self) -> bool {
self.supervisors().iter().any(Supervisor::is_connected)
}
pub fn stopped(&self) -> watch::Receiver<bool> {
self.shared.stopping.subscribe()
}
pub async fn borrow(&self) -> Result<ResourceLease<L>, Error<L::Error>> {
self.begin();
let mut lease = borrow(self.shared.clone()).await?;
lease.owner = Some(self.owner.clone());
Ok(lease)
}
pub async fn recover(
&self,
expected: Option<&ResourceHandle<L::Resource>>,
terminal: bool,
) -> Result<(), Error<L::Error>> {
for supervisor in self.supervisors() {
if let Some(handle) = expected {
supervisor.recover(handle, terminal).await?;
} else if let Some(handle) = supervisor.current() {
supervisor.recover(&handle, terminal).await?;
}
}
Ok(())
}
pub fn set_attribute(&self, name: impl Into<String>, setter: AttributeSetter<L::Resource>) {
let name = name.into();
let mut contents = self.shared.contents.lock().unwrap();
self.shared.apply_attribute(&contents, &name, &setter);
contents.attributes.insert(name, setter);
}
pub fn set_value<T: Clone + Send + Sync + 'static>(
&self,
name: impl Into<String>,
value: T,
apply: impl Fn(&L::Resource, &T) + Send + Sync + 'static,
) {
let name = name.into();
let value = Arc::new(value);
let stored = value.clone();
let setter: AttributeSetter<L::Resource> =
Arc::new(move |resource| apply(resource, &value));
let mut contents = self.shared.contents.lock().unwrap();
self.shared.apply_attribute(&contents, &name, &setter);
contents.values.insert(name.clone(), stored);
contents.attributes.insert(name, setter);
}
pub fn value<T: Clone + Send + Sync + 'static>(&self, name: &str) -> Option<T> {
self.shared
.contents
.lock()
.unwrap()
.values
.get(name)?
.downcast_ref::<T>()
.cloned()
}
pub fn set_on_resource_created(&self, hook: CreatedHook<L::Resource, L::Error>) {
*self.shared.on_created.lock().unwrap() = Some(hook);
}
pub async fn execute<F, Fut, T>(&self, operation: F) -> Result<T, Error<L::Error>>
where
F: FnOnce(Arc<L::Resource>) -> Fut + Send + 'static,
Fut: Future<Output = Result<T, L::Error>> + Send + 'static,
T: Send + 'static,
{
self.execute_with_priority(0, operation).await
}
pub async fn execute_with_priority<F, Fut, T>(
&self,
priority: i32,
operation: F,
) -> Result<T, Error<L::Error>>
where
F: FnOnce(Arc<L::Resource>) -> Fut + Send + 'static,
Fut: Future<Output = Result<T, L::Error>> + Send + 'static,
T: Send + 'static,
{
self.enqueue(
priority,
|lease| async move { lease.execute(operation).await },
)
.await
}
pub async fn execute_with_retry<F, Fut, T, P>(
&self,
policy: RetryPolicy,
operation: F,
retry_if: P,
) -> Result<T, Error<L::Error>>
where
F: FnMut(Arc<L::Resource>) -> Fut + Send + 'static,
Fut: Future<Output = Result<T, L::Error>> + Send + 'static,
T: Send + 'static,
P: FnMut(&L::Error) -> bool + Send + 'static,
{
self.execute_with_priority_and_retry(0, policy, operation, retry_if)
.await
}
pub async fn execute_with_priority_and_retry<F, Fut, T, P>(
&self,
priority: i32,
policy: RetryPolicy,
operation: F,
retry_if: P,
) -> Result<T, Error<L::Error>>
where
F: FnMut(Arc<L::Resource>) -> Fut + Send + 'static,
Fut: Future<Output = Result<T, L::Error>> + Send + 'static,
T: Send + 'static,
P: FnMut(&L::Error) -> bool + Send + 'static,
{
if policy.timeout.is_zero() {
return Err(Error::Timeout);
}
let queued = self.enqueue(priority, move |lease| async move {
lease.execute_with_retry(policy, operation, retry_if).await
});
tokio::time::timeout(policy.timeout, queued)
.await
.unwrap_or(Err(Error::Timeout))
}
async fn enqueue<F, Fut, T>(&self, priority: i32, operation: F) -> Result<T, Error<L::Error>>
where
F: FnOnce(ResourceLease<L>) -> Fut + Send + 'static,
Fut: Future<Output = Result<T, Error<L::Error>>> + Send + 'static,
T: Send + 'static,
{
self.begin();
let (sender, receiver) = oneshot::channel();
let mut cancellation = {
let mut contents = self.shared.contents.lock().unwrap();
if *self.shared.stopping.borrow() {
return Err(Error::Stopped);
}
let sequence = contents.sequence;
contents.sequence += 1;
let lease = if self.shared.fast_admission
&& contents.queue.is_empty()
&& self.shared.waiters.load(Ordering::Acquire) == 0
{
reserve_ready(&self.shared, &contents)
} else {
None
};
if let Some(lease) = lease {
self.shared
.operations
.lock()
.unwrap()
.spawn(run_job(sender, lease, operation));
self.shared.dispatcher.notify_one();
None
} else {
let cancelled = Arc::new(AtomicBool::new(false));
let cancellation = CancelOnDrop {
cancelled: cancelled.clone(),
dispatcher: self.shared.dispatcher.clone(),
armed: true,
};
contents
.queue
.push(Operation {
priority,
sequence,
cancelled,
job: Box::new(move |lease| Box::pin(run_job(sender, lease, operation))),
})
.map_err(Error::Queue)?;
self.shared.dispatcher.notify_one();
Some(cancellation)
}
};
let result = receiver.await.unwrap_or(Err(Error::Stopped));
if let Some(cancellation) = &mut cancellation {
cancellation.armed = false;
}
drop(cancellation);
result
}
pub async fn stop(&self) {
self.shared.stopping.send_replace(true);
self.shared.notify();
if !self.owner.started.load(Ordering::Acquire) {
self.shared.done.send_replace(true);
}
let mut done = self.shared.done.subscribe();
while !*done.borrow_and_update() {
if done.changed().await.is_err() {
return;
}
}
}
}
async fn run_job<L, F, Fut, T>(
mut sender: oneshot::Sender<Result<T, Error<L::Error>>>,
lease: ResourceLease<L>,
operation: F,
) where
L: Lifecycle,
F: FnOnce(ResourceLease<L>) -> Fut + Send + 'static,
Fut: Future<Output = Result<T, Error<L::Error>>> + Send + 'static,
T: Send + 'static,
{
if sender.is_closed() {
return;
}
tokio::select! {
biased;
_ = sender.closed() => {},
result = operation(lease) => { let _ = sender.send(result); }
}
}
struct CancelOnDrop {
cancelled: Arc<AtomicBool>,
dispatcher: Arc<Notify>,
armed: bool,
}
impl Drop for CancelOnDrop {
fn drop(&mut self) {
if !self.armed {
return;
}
self.cancelled.store(true, Ordering::Release);
self.dispatcher.notify_one();
}
}
pub struct ResourceLease<L: Lifecycle> {
entry: Arc<Entry<L>>,
shared: Arc<Shared<L>>,
owner: Option<Arc<PoolOwner>>,
}
impl<L: Lifecycle> ResourceLease<L> {
pub fn release(self) {}
pub fn supervisor(&self) -> &Supervisor<L> {
&self.entry.supervisor
}
pub async fn acquire(&self) -> Result<ResourceHandle<L::Resource>, Error<L::Error>> {
if *self.shared.stopping.borrow() {
return Err(Error::Stopped);
}
self.entry.supervisor.acquire().await
}
pub async fn execute<F, Fut, T>(&self, operation: F) -> Result<T, Error<L::Error>>
where
F: FnOnce(Arc<L::Resource>) -> Fut,
Fut: Future<Output = Result<T, L::Error>>,
{
let mut stopping = self.shared.stopping.subscribe();
tokio::select! { biased; _ = crate::shutdown(&mut stopping) => Err(Error::Stopped), result = self.entry.supervisor.execute(operation) => result }
}
pub async fn execute_with_retry<F, Fut, T, P>(
&self,
policy: RetryPolicy,
operation: F,
retry_if: P,
) -> Result<T, Error<L::Error>>
where
F: FnMut(Arc<L::Resource>) -> Fut,
Fut: Future<Output = Result<T, L::Error>>,
P: FnMut(&L::Error) -> bool,
{
let mut stopping = self.shared.stopping.subscribe();
tokio::select! {
biased;
_ = crate::shutdown(&mut stopping) => Err(Error::Stopped),
result = self.entry.supervisor.execute_with_retry(policy, operation, retry_if) => result,
}
}
}
impl<L: Lifecycle> Drop for ResourceLease<L> {
fn drop(&mut self) {
*self.entry.idle_since.lock().unwrap() = Instant::now();
self.entry.leased.store(false, Ordering::Release);
self.shared.notify();
}
}
async fn borrow<L: Lifecycle>(shared: Arc<Shared<L>>) -> Result<ResourceLease<L>, Error<L::Error>> {
shared.waiters.fetch_add(1, Ordering::AcqRel);
let _waiter = Waiter(shared.clone());
let mut changed = shared.changed.subscribe();
let mut stopping = shared.stopping.subscribe();
loop {
if let Some(lease) = try_borrow(&shared)? {
return Ok(lease);
}
tokio::select! { biased; _ = crate::shutdown(&mut stopping) => return Err(Error::Stopped), _ = changed.changed() => {} }
}
}
fn try_borrow<L: Lifecycle>(
shared: &Arc<Shared<L>>,
) -> Result<Option<ResourceLease<L>>, Error<L::Error>> {
let mut contents = shared.contents.lock().unwrap();
if *shared.stopping.borrow() {
return Err(Error::Stopped);
}
if let Some(lease) = reserve_ready(shared, &contents) {
return Ok(Some(lease));
}
let demand = shared.waiters.load(Ordering::Acquire)
+ contents.queue.len()
+ contents
.entries
.iter()
.filter(|entry| entry.leased.load(Ordering::Acquire))
.count();
if contents.entries.len() + contents.retiring < shared.config.max_size.min(demand) {
shared.grow(&mut contents);
}
Ok(None)
}
fn reserve_ready<L: Lifecycle>(
shared: &Arc<Shared<L>>,
contents: &Contents<L>,
) -> Option<ResourceLease<L>> {
for entry in &contents.entries {
if entry.supervisor.state() == ResourceState::Connected
&& entry
.leased
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
{
return Some(ResourceLease {
entry: entry.clone(),
shared: shared.clone(),
owner: None,
});
}
}
None
}
struct Waiter<L: Lifecycle>(Arc<Shared<L>>);
impl<L: Lifecycle> Drop for Waiter<L> {
fn drop(&mut self) {
self.0.waiters.fetch_sub(1, Ordering::AcqRel);
self.0.dispatcher.notify_one();
}
}
async fn run<L: Lifecycle>(shared: Arc<Shared<L>>) {
let mut stopping = shared.stopping.subscribe();
let mut retiring = JoinSet::new();
let mut reaper = Box::pin(sleep(
shared.config.idle_timeout.min(Duration::from_secs(1)),
));
loop {
let queued = {
let mut contents = shared.contents.lock().unwrap();
contents.queue.retain(&mut |item| !item.is_cancelled());
!contents.queue.is_empty()
};
let lease = if queued {
try_borrow(&shared).unwrap_or(None)
} else {
None
};
tokio::select! {
biased;
_ = crate::shutdown(&mut stopping) => break,
lease = async { match lease { Some(lease) => lease, None => std::future::pending().await } } => {
let item = shared.contents.lock().unwrap().queue.pop();
if let Some(item) = item && !item.cancelled.load(Ordering::Acquire) { shared.operations.lock().unwrap().spawn((item.job)(lease)); }
}
_ = shared.dispatcher.notified() => {},
_ = std::future::poll_fn(|cx| shared.operations.lock().unwrap().poll_join_next(cx)),
if !shared.operations.lock().unwrap().is_empty() => {},
_ = retiring.join_next(), if !retiring.is_empty() => {
shared.contents.lock().unwrap().retiring -= 1; shared.notify();
}
_ = &mut reaper => {
let mut contents = shared.contents.lock().unwrap();
if contents.queue.is_empty() && shared.waiters.load(Ordering::Acquire) == 0 {
let mut index = 0;
while index < contents.entries.len() && contents.entries.len() > shared.config.min_size {
let entry = &contents.entries[index];
if !entry.leased.load(Ordering::Acquire) && entry.idle_since.lock().unwrap().elapsed() >= shared.config.idle_timeout {
let entry = contents.entries.remove(index); contents.retiring += 1;
retiring.spawn(stop_entry(entry)); shared.notify();
} else { index += 1; }
}
}
reaper.as_mut().reset(Instant::now() + shared.config.idle_timeout.min(Duration::from_secs(1)));
}
}
}
let entries = {
let mut contents = shared.contents.lock().unwrap();
contents.queue.clear();
contents.entries.clone()
};
shared.operations.lock().unwrap().abort_all();
while std::future::poll_fn(|cx| shared.operations.lock().unwrap().poll_join_next(cx))
.await
.is_some()
{}
for entry in entries {
retiring.spawn(stop_entry(entry));
}
while retiring.join_next().await.is_some() {}
shared.done.send_replace(true);
shared.notify();
}
async fn stop_entry<L: Lifecycle>(entry: Arc<Entry<L>>) {
entry.supervisor.stop().await;
let relay = entry.relay.lock().unwrap().take();
if let Some(relay) = relay {
let _ = relay.await;
}
}
pub struct ManagedResourceProxy<L: Lifecycle> {
pub pool: Pool<L>,
}
impl<L: Lifecycle> Clone for ManagedResourceProxy<L> {
fn clone(&self) -> Self {
Self {
pool: self.pool.clone(),
}
}
}
impl<L: Lifecycle> ManagedResourceProxy<L> {
pub fn new(pool: Pool<L>) -> Self {
Self { pool }
}
pub async fn execute<F, Fut, T>(&self, operation: F) -> Result<T, Error<L::Error>>
where
F: FnOnce(Arc<L::Resource>) -> Fut + Send + 'static,
Fut: Future<Output = Result<T, L::Error>> + Send + 'static,
T: Send + 'static,
{
self.pool.execute(operation).await
}
pub async fn execute_with_retry<F, Fut, T, P>(
&self,
policy: RetryPolicy,
operation: F,
retry_if: P,
) -> Result<T, Error<L::Error>>
where
F: FnMut(Arc<L::Resource>) -> Fut + Send + 'static,
Fut: Future<Output = Result<T, L::Error>> + Send + 'static,
T: Send + 'static,
P: FnMut(&L::Error) -> bool + Send + 'static,
{
self.pool
.execute_with_retry(policy, operation, retry_if)
.await
}
pub async fn connected(&self) -> Result<(), Error<L::Error>> {
self.pool.connected().await
}
pub fn is_connected(&self) -> bool {
self.pool.is_connected()
}
pub fn set_attribute(&self, name: impl Into<String>, setter: AttributeSetter<L::Resource>) {
self.pool.set_attribute(name, setter);
}
pub fn attribute<T>(&self, read: impl FnOnce(&L::Resource) -> T) -> Option<T> {
self.pool.with_latest(read)
}
pub fn set_value<T: Clone + Send + Sync + 'static>(
&self,
name: impl Into<String>,
value: T,
apply: impl Fn(&L::Resource, &T) + Send + Sync + 'static,
) {
self.pool.set_value(name, value, apply);
}
pub fn value<T: Clone + Send + Sync + 'static>(&self, name: &str) -> Option<T> {
self.pool.value(name)
}
pub async fn stop(&self) {
self.pool.stop().await;
}
}
#[macro_export]
macro_rules! managed_client {
($visibility:vis struct $name:ident for $lifecycle:ty { $($methods:tt)* }) => {
#[derive(Clone)]
$visibility struct $name<B = $crate::ManagedResourceProxy<$lifecycle>> { pub managed: B }
impl $name<$crate::ManagedResourceProxy<$lifecycle>> {
pub fn new(pool: $crate::Pool<$lifecycle>) -> Self { Self { managed: $crate::ManagedResourceProxy::new(pool) } }
}
impl $name<$crate::SupervisedResourceProxy<$lifecycle>> {
pub fn from_supervisor(supervisor: $crate::Supervisor<$lifecycle>) -> Self {
Self { managed: $crate::SupervisedResourceProxy::new(supervisor) }
}
}
impl<B: $crate::ManagedClientBackend<$lifecycle>> $name<B> {
$crate::managed_client_methods! { $lifecycle; $($methods)* }
}
};
}
#[doc(hidden)]
#[macro_export]
macro_rules! managed_client_methods {
($lifecycle:ty;) => {};
($lifecycle:ty; async fn $method:ident($($argument:ident: $ty:ty),* $(,)?) -> $output:ty; $($rest:tt)*) => {
pub async fn $method(&self, $($argument: $ty),*) -> Result<$output, $crate::Error<<$lifecycle as $crate::Lifecycle>::Error>> {
$crate::ManagedClientBackend::execute(&self.managed, move |resource| async move { resource.$method($($argument),*).await }).await
}
$crate::managed_client_methods! { $lifecycle; $($rest)* }
};
($lifecycle:ty; fn $method:ident($($argument:ident: $ty:ty),* $(,)?) -> $output:ty; $($rest:tt)*) => {
pub async fn $method(&self, $($argument: $ty),*) -> Result<$output, $crate::Error<<$lifecycle as $crate::Lifecycle>::Error>> {
$crate::ManagedClientBackend::execute(&self.managed, move |resource| async move { resource.$method($($argument),*) }).await
}
$crate::managed_client_methods! { $lifecycle; $($rest)* }
};
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn relay_forwards_readiness_published_before_its_first_poll() {
let (state, receiver) = watch::channel(crate::ResourceStatus::<()> {
state: ResourceState::Connecting,
handle: None,
});
let (changed, mut observer) = watch::channel(0_u64);
let dispatcher = Arc::new(Notify::new());
state.send_replace(crate::ResourceStatus {
state: ResourceState::Connected,
handle: None,
});
let relay = tokio::spawn(relay_state(receiver, changed, dispatcher.clone()));
tokio::time::timeout(Duration::from_secs(1), observer.changed())
.await
.unwrap()
.unwrap();
tokio::time::timeout(Duration::from_secs(1), dispatcher.notified())
.await
.unwrap();
state.send_replace(crate::ResourceStatus {
state: ResourceState::Stopped,
handle: None,
});
tokio::time::timeout(Duration::from_secs(1), relay)
.await
.unwrap()
.unwrap();
assert_eq!(*observer.borrow(), 2);
}
struct Hooks;
#[crate::async_trait]
impl Lifecycle for Hooks {
type Resource = ();
type Error = std::convert::Infallible;
async fn create(&self) -> Result<(), Self::Error> {
Ok(())
}
async fn connect(&self, _: &()) -> Result<(), Self::Error> {
Ok(())
}
async fn disconnect(&self, _: &()) -> Result<(), Self::Error> {
Ok(())
}
}
#[tokio::test]
async fn shutdown_joins_notification_relays_even_when_supervisors_are_retained() {
let pool = Pool::start(
|| Supervisor::new(Hooks, crate::Config::default()),
PoolConfig::default(),
);
pool.connected().await.unwrap();
let retained = pool.supervisors();
pool.stop().await;
assert!(
pool.shared
.contents
.lock()
.unwrap()
.entries
.iter()
.all(|entry| entry.relay.lock().unwrap().is_none())
);
assert!(
retained
.iter()
.all(|supervisor| supervisor.state() == ResourceState::Stopped)
);
}
#[tokio::test]
async fn notification_relays_are_joined_during_idle_retirement() {
let pool = Pool::start(
|| Supervisor::new(Hooks, crate::Config::default()),
PoolConfig {
min_size: 0,
idle_timeout: Duration::from_millis(5),
..PoolConfig::default()
},
);
let lease = pool.borrow().await.unwrap();
let entry = lease.entry.clone();
drop(lease);
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if entry.relay.lock().unwrap().is_none() {
break;
}
sleep(Duration::from_millis(1)).await;
}
})
.await
.unwrap();
pool.stop().await;
}
struct PendingHooks;
#[crate::async_trait]
impl Lifecycle for PendingHooks {
type Resource = ();
type Error = std::convert::Infallible;
async fn create(&self) -> Result<(), Self::Error> {
Ok(())
}
async fn connect(&self, _: &()) -> Result<(), Self::Error> {
std::future::pending().await
}
async fn disconnect(&self, _: &()) -> Result<(), Self::Error> {
Ok(())
}
fn watch_disconnect(&self, _: Arc<()>) -> Option<crate::LifecycleFuture<Self::Error>> {
Some(Box::pin(std::future::pending()))
}
}
#[tokio::test]
async fn shutdown_joins_relays_while_connecting_with_watchdog_configured() {
let pool = Pool::start(
|| Supervisor::new(PendingHooks, crate::Config::default()),
PoolConfig::default(),
);
tokio::time::timeout(Duration::from_secs(1), async {
while pool.state() != ResourceState::Connecting {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
let entries = pool.shared.contents.lock().unwrap().entries.clone();
assert_eq!(entries.len(), 1);
assert!(entries[0].relay.lock().unwrap().is_some());
tokio::time::timeout(Duration::from_secs(1), pool.stop())
.await
.unwrap();
assert!(entries[0].relay.lock().unwrap().is_none());
assert_eq!(entries[0].supervisor.state(), ResourceState::Stopped);
assert!(*pool.shared.done.borrow());
}
}