use futures::stream::{self, StreamExt};
use kdam::{BarExt, tqdm};
use tracing::debug;
use crate::{BamlType, Facet, PredictError, Predicted};
type IndexedForwardResult<T> = (usize, Result<Predicted<T>, PredictError>);
#[allow(async_fn_in_trait)]
pub trait Module: Send + Sync {
type Input: BamlType + for<'a> Facet<'a> + Send + Sync;
type Output: BamlType + for<'a> Facet<'a> + Send + Sync;
async fn forward(&self, input: Self::Input) -> Result<Predicted<Self::Output>, PredictError>;
async fn call(&self, input: Self::Input) -> Result<Predicted<Self::Output>, PredictError> {
self.forward(input).await
}
}
#[tracing::instrument(
name = "dsrs.forward_all",
level = "debug",
skip(module, inputs),
fields(total_inputs = inputs.len(), max_concurrency)
)]
pub async fn forward_all<M>(
module: &M,
inputs: Vec<M::Input>,
max_concurrency: usize,
) -> Vec<Result<Predicted<M::Output>, PredictError>>
where
M: Module + ?Sized,
{
forward_all_with_progress(module, inputs, max_concurrency, true).await
}
#[tracing::instrument(
name = "dsrs.forward_all_with_progress",
level = "debug",
skip(module, inputs),
fields(total_inputs = inputs.len(), max_concurrency, display_progress)
)]
pub async fn forward_all_with_progress<M>(
module: &M,
inputs: Vec<M::Input>,
max_concurrency: usize,
display_progress: bool,
) -> Vec<Result<Predicted<M::Output>, PredictError>>
where
M: Module + ?Sized,
{
let total = inputs.len();
let mut pb = if display_progress {
Some(tqdm!(total = total, desc = "Processing"))
} else {
None
};
let mut indexed_results: Vec<IndexedForwardResult<M::Output>> =
stream::iter(inputs.into_iter().enumerate())
.map(|(idx, input)| async move { (idx, module.call(input).await) })
.buffer_unordered(max_concurrency)
.inspect(|_| {
if let Some(ref mut progress) = pb {
let _ = progress.update(1);
}
})
.collect()
.await;
indexed_results.sort_by_key(|(idx, _)| *idx);
let mut outcomes = Vec::with_capacity(indexed_results.len());
for (_, outcome) in indexed_results {
outcomes.push(outcome);
}
debug!(outcomes = outcomes.len(), "forward_all completed");
outcomes
}