use crate::util::cancel;
use anyhow::Result;
use std::{future::Future, path::PathBuf};
use tokio::{sync, task::JoinSet};
use tokio_util::sync::CancellationToken;
#[derive(Debug, Clone, Copy, Default)]
pub struct BatchSummary {
pub succeeded: usize,
pub failed: usize,
pub termination: BatchTermination,
}
impl BatchSummary {
pub fn was_cancelled(&self) -> bool {
matches!(self.termination, BatchTermination::Cancelled)
}
pub fn stopped_on_error(&self) -> bool {
matches!(self.termination, BatchTermination::StoppedOnError)
}
pub fn stopped_early(&self) -> bool {
self.was_cancelled() || self.stopped_on_error()
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum BatchTermination {
#[default]
Completed,
Cancelled,
StoppedOnError,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum FailurePolicy {
#[default]
Continue,
CancelRemaining,
}
#[derive(Debug)]
pub enum BatchEvent {
Started(usize),
Done(usize, WorkResult),
AllDone(BatchSummary),
}
#[derive(Debug, Clone)]
pub enum WorkResult {
Success(PathBuf),
Failed(String),
Cancelled,
}
impl WorkResult {
pub fn from_result(result: Result<PathBuf>) -> Self {
match result {
Ok(path) => Self::Success(path),
Err(error) if cancel::is_cancelled(&error) => Self::Cancelled,
Err(error) => Self::Failed(error.to_string()),
}
}
pub fn is_success(&self) -> bool {
matches!(self, Self::Success(_))
}
pub fn is_failure(&self) -> bool {
matches!(self, Self::Failed(_))
}
pub fn is_cancelled(&self) -> bool {
matches!(self, Self::Cancelled)
}
pub fn path(&self) -> Option<&PathBuf> {
match self {
Self::Success(path) => Some(path),
Self::Failed(_) | Self::Cancelled => None,
}
}
pub fn error(&self) -> Option<&str> {
match self {
Self::Failed(error) => Some(error),
Self::Success(_) | Self::Cancelled => None,
}
}
}
pub async fn run_parallel<F, Fut>(
inputs: Vec<PathBuf>,
concurrency: usize,
cancel: CancellationToken,
failure_policy: FailurePolicy,
event_tx: sync::mpsc::UnboundedSender<BatchEvent>,
Worker: F,
) -> BatchSummary
where
F: Fn(PathBuf, CancellationToken) -> Fut + Send + Sync + Clone + 'static,
Fut: Future<Output = Result<PathBuf>> + Send + 'static,
{
let limit = concurrency.max(1);
let batch_cancel = cancel.child_token();
let mut pending = inputs.into_iter().enumerate();
let mut tasks = JoinSet::new();
let mut succeeded = 0usize;
let mut failed = 0usize;
let mut stopped_on_error = false;
while tasks.len() < limit {
if !spawn_next(&mut tasks, &mut pending, &batch_cancel, &event_tx, &Worker) {
break;
}
}
while let Some(join_result) = tasks.join_next().await {
match join_result {
Ok((_, WorkResult::Success(_))) => {
succeeded += 1;
}
Ok((_, WorkResult::Failed(_))) => {
failed += 1;
if failure_policy == FailurePolicy::CancelRemaining && !batch_cancel.is_cancelled()
{
stopped_on_error = true;
batch_cancel.cancel();
}
}
Ok((_, WorkResult::Cancelled)) => {}
Err(_) => {
failed += 1;
if failure_policy == FailurePolicy::CancelRemaining && !batch_cancel.is_cancelled()
{
stopped_on_error = true;
batch_cancel.cancel();
}
}
}
while !batch_cancel.is_cancelled() && tasks.len() < limit {
if !spawn_next(&mut tasks, &mut pending, &batch_cancel, &event_tx, &Worker) {
break;
}
}
}
let summary = BatchSummary {
succeeded,
failed,
termination: if cancel.is_cancelled() {
BatchTermination::Cancelled
} else if stopped_on_error {
BatchTermination::StoppedOnError
} else {
BatchTermination::Completed
},
};
let _ = event_tx.send(BatchEvent::AllDone(summary));
summary
}
pub fn num_cpus() -> usize {
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1)
}
fn spawn_next<F, Fut>(
tasks: &mut JoinSet<(usize, WorkResult)>,
pending: &mut impl Iterator<Item = (usize, PathBuf)>,
cancel: &CancellationToken,
event_tx: &sync::mpsc::UnboundedSender<BatchEvent>,
Worker: &F,
) -> bool
where
F: Fn(PathBuf, CancellationToken) -> Fut + Send + Sync + Clone + 'static,
Fut: Future<Output = Result<PathBuf>> + Send + 'static,
{
if cancel.is_cancelled() {
return false;
}
let Some((index, input)) = pending.next() else {
return false;
};
let child_cancel = cancel.child_token();
let tx = event_tx.clone();
let Worker = Worker.clone();
tasks.spawn(async move {
let _ = tx.send(BatchEvent::Started(index));
let result = WorkResult::from_result(Worker(input, child_cancel).await);
let _ = tx.send(BatchEvent::Done(index, result.clone()));
(index, result)
});
true
}
#[cfg(test)]
mod tests {
use super::*;
use crate::util::cancel::Cancelled;
#[test]
fn work_result_keeps_success_path() {
let result = WorkResult::from_result(Ok(PathBuf::from("out.mp4")));
assert!(matches!(result, WorkResult::Success(path) if path == PathBuf::from("out.mp4")));
}
#[test]
fn work_result_detects_cancellation() {
let result = WorkResult::from_result(Err(Cancelled.into()));
assert!(result.is_cancelled());
}
}