use std::panic::AssertUnwindSafe;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::sync::Weak;
use std::time::{Duration, Instant};
use futures_util::FutureExt;
mod worker_pool;
use super::{ExecutionRuntime, RuntimeComponents, ScheduledWork};
use crate::runtime::sync::SubmissionGuard;
use crate::transfer::{TransferId, WorkOutcome};
use worker_pool::WorkerPool;
#[derive(Debug)]
pub(crate) struct TokioMultiThreadRuntime {
pool: Arc<WorkerPool>,
handle: Weak<crate::client::Handle>,
worker_count: AtomicUsize,
components: RuntimeComponents,
}
impl TokioMultiThreadRuntime {
pub(crate) fn new(handle: Weak<crate::client::Handle>) -> Self {
let components = if cfg!(target_family = "wasm") {
RuntimeComponents::inline_io()
} else {
RuntimeComponents::default()
};
Self {
pool: Arc::new(WorkerPool::new()),
handle,
worker_count: AtomicUsize::new(0),
components,
}
}
fn ensure_workers_started(&self) {
if self.pool.mark_started() {
let target = self
.handle
.upgrade()
.expect("Handle dropped")
.controller
.target();
if !cfg!(target_family = "wasm")
&& matches!(
tokio::runtime::Handle::try_current().map(|h| h.runtime_flavor()),
Ok(tokio::runtime::RuntimeFlavor::CurrentThread)
)
{
tracing::warn!(
target: crate::telemetry::TARGET_SCHEDULING,
"Caller Tokio execution selected on a current-thread runtime; \
transfer workers will serialize onto one thread. Use a \
multi-threaded runtime or owned managed execution."
);
}
self.spawn_workers_raw(target);
self.worker_count.store(target, Ordering::Release);
}
}
fn ensure_worker_capacity(&self) {
if !self.pool.is_started() {
return;
}
let target = self
.handle
.upgrade()
.expect("Handle dropped")
.controller
.target();
let current = self.worker_count.load(Ordering::Acquire);
if target > current
&& self
.worker_count
.compare_exchange(current, target, Ordering::Release, Ordering::Relaxed)
.is_ok()
{
self.spawn_workers_raw(target - current);
tracing::debug!(target: crate::telemetry::TARGET_SCHEDULING, old = current, new = target, "spawning additional workers");
}
}
fn spawn_workers_raw(&self, count: usize) {
for _ in 0..count {
let pool = Arc::clone(&self.pool);
let handle = self.handle.clone();
tokio::spawn(async move {
worker_loop(pool, handle).await;
});
}
}
}
impl TokioMultiThreadRuntime {
#[cfg(test)]
pub(crate) fn worker_count(&self) -> usize {
self.worker_count.load(Ordering::Acquire)
}
}
impl ExecutionRuntime for TokioMultiThreadRuntime {
fn dispatch(&self, batch: &mut SubmissionGuard<'_, ScheduledWork>) {
self.ensure_workers_started();
self.ensure_worker_capacity();
for work in batch.drain() {
self.pool.push(work);
}
}
fn shutdown(&self) {
self.pool.shutdown();
}
fn shutdown_completion(&self) -> Option<crate::ShutdownCompletion> {
None
}
fn remove_pending_for_transfer(&self, id: TransferId) -> usize {
self.pool.remove_for_transfer(id)
}
fn components(&self) -> &RuntimeComponents {
&self.components
}
}
async fn worker_loop(pool: Arc<WorkerPool>, handle: Weak<crate::client::Handle>) {
static WORKER_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let wid = WORKER_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
loop {
let Some(mut work) = pool.next_work().await else {
tracing::debug!(target: crate::telemetry::TARGET_EXECUTION, wid, "shutdown");
break;
};
let Some(h) = handle.upgrade() else {
break;
};
h.scheduler.on_dispatch();
let tid = work.descriptor.id();
work.descriptor.work_started();
if work.descriptor.is_terminal() {
tracing::trace!(target: crate::telemetry::TARGET_EXECUTION, wid, %tid, "skipped (terminal)");
pool.complete();
h.scheduler
.on_completion(work, WorkOutcome::Cancelled, Duration::ZERO);
continue;
}
tracing::trace!(target: crate::telemetry::TARGET_EXECUTION, wid, %tid, "executing");
let transfer = work.descriptor.transfer();
let started = Instant::now();
let token = work.descriptor.cancellation_token().clone();
let outcome = AssertUnwindSafe(async {
tokio::select! {
biased;
_ = token.cancelled() => WorkOutcome::Cancelled,
outcome = transfer.execute(&mut work.item) => outcome,
}
})
.catch_unwind()
.await;
let outcome = match outcome {
Ok(outcome) => outcome,
Err(_panic) => {
tracing::error!(target: crate::telemetry::TARGET_EXECUTION, wid, %tid, "panic in transfer execute");
pool.complete();
h.scheduler.on_panic(work);
continue;
}
};
tracing::trace!(target: crate::telemetry::TARGET_EXECUTION, wid, %tid, ?outcome, "completed");
let elapsed = started.elapsed();
pool.complete();
h.scheduler.on_completion(work, outcome, elapsed);
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::client::Handle;
use crate::scheduler::ConcurrencyController;
use std::sync::atomic::AtomicUsize as StdAtomicUsize;
use std::sync::OnceLock;
#[derive(Debug)]
struct AdjustableConcurrency(StdAtomicUsize);
impl AdjustableConcurrency {
fn new(target: usize) -> Self {
Self(StdAtomicUsize::new(target))
}
fn set_target(&self, target: usize) {
self.0.store(target, std::sync::atomic::Ordering::Release);
}
}
impl ConcurrencyController for AdjustableConcurrency {
fn target(&self) -> usize {
self.0.load(std::sync::atomic::Ordering::Acquire)
}
}
fn test_config() -> crate::Config {
let s3_client = aws_smithy_mocks::mock_client!(aws_sdk_s3, []);
crate::Config::builder()
.sdk_client(s3_client)
.build()
.expect("valid config")
}
fn build_handle_with_adjustable_concurrency(
initial_target: usize,
) -> (
Arc<Handle>,
Arc<TokioMultiThreadRuntime>,
Arc<AdjustableConcurrency>,
) {
let controller = Arc::new(AdjustableConcurrency::new(initial_target));
let slot: Arc<OnceLock<Arc<TokioMultiThreadRuntime>>> = Arc::new(OnceLock::new());
let slot_clone = slot.clone();
let handle = Handle::new_for_test_with_runtime(test_config(), controller.clone(), |weak| {
let rt = Arc::new(TokioMultiThreadRuntime::new(weak));
slot_clone.set(rt.clone()).ok();
rt
});
let runtime = slot.get().unwrap().clone();
(handle, runtime, controller)
}
#[cfg_attr(miri, ignore)]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn genesis_spawns_exact_target_workers() {
let (_handle, runtime, _controller) = build_handle_with_adjustable_concurrency(4);
assert_eq!(
runtime.worker_count(),
0,
"no workers before first dispatch"
);
let sq = crate::runtime::sync::SubmissionQueue::new(8);
let sub = sq.enter();
if let Some(mut guard) = sub.submit() {
runtime.dispatch(&mut guard);
}
assert_eq!(
runtime.worker_count(),
4,
"genesis must spawn exactly target workers"
);
}
#[cfg_attr(miri, ignore)]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn growth_does_not_double_count() {
let (_handle, runtime, controller) = build_handle_with_adjustable_concurrency(4);
let sq = crate::runtime::sync::SubmissionQueue::new(8);
let sub = sq.enter();
if let Some(mut guard) = sub.submit() {
runtime.dispatch(&mut guard);
}
assert_eq!(runtime.worker_count(), 4);
controller.set_target(8);
let sub = sq.enter();
if let Some(mut guard) = sub.submit() {
runtime.dispatch(&mut guard);
}
assert_eq!(
runtime.worker_count(),
8,
"after growth 4 -> 8, counter must be 8 (not 12)"
);
}
}