use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Mutex, PoisonError};
use futures::future::try_join_all;
use crate::error::TesseraError;
use crate::models::PartitionRef;
use crate::readers::ResolvedPartition;
pub const MAX_WORKERS: usize = 8;
pub fn resolve_sync(
fetch_urls: impl Fn(&PartitionRef) -> Result<Vec<String>, TesseraError> + Sync,
refs: &[PartitionRef],
) -> Result<Vec<ResolvedPartition>, TesseraError> {
if let [partition] = refs {
return Ok(vec![(partition.clone(), fetch_urls(partition)?)]);
}
let workers = MAX_WORKERS.min(refs.len());
let next_index = AtomicUsize::new(0);
let urls: Mutex<Vec<Option<Vec<String>>>> = Mutex::new(vec![None; refs.len()]);
let first_error: Mutex<Option<TesseraError>> = Mutex::new(None);
std::thread::scope(|scope| {
for _ in 0..workers {
scope.spawn(|| {
loop {
if first_error
.lock()
.unwrap_or_else(PoisonError::into_inner)
.is_some()
{
break;
}
let index = next_index.fetch_add(1, Ordering::Relaxed);
if index >= refs.len() {
break;
}
match fetch_urls(&refs[index]) {
Ok(url) => {
urls.lock().unwrap_or_else(PoisonError::into_inner)[index] = Some(url);
}
Err(err) => {
let mut guard =
first_error.lock().unwrap_or_else(PoisonError::into_inner);
if guard.is_none() {
*guard = Some(err);
}
break;
}
}
}
});
}
});
if let Some(err) = first_error
.into_inner()
.unwrap_or_else(PoisonError::into_inner)
{
return Err(err);
}
let resolved = urls.into_inner().unwrap_or_else(PoisonError::into_inner);
Ok(refs
.iter()
.zip(resolved)
.map(|(partition, url)| {
(
partition.clone(),
url.expect("resolved when no error recorded"),
)
})
.collect())
}
pub async fn resolve_async<F, Fut>(
fetch_urls: F,
refs: &[PartitionRef],
) -> Result<Vec<ResolvedPartition>, TesseraError>
where
F: Fn(PartitionRef) -> Fut,
Fut: Future<Output = Result<Vec<String>, TesseraError>>,
{
let urls = try_join_all(refs.iter().map(|partition| fetch_urls(partition.clone()))).await?;
Ok(refs.iter().cloned().zip(urls).collect())
}