use super::{OwnedDataKey, PageStore, PendingPage};
use crate::{ObjectLogError, native_v2::data::DataFrame};
use std::{future::Future, pin::Pin, task::Poll};
pub(in crate::native_v2) const UPLOAD_CONCURRENCY: usize = 8;
pub(in crate::native_v2) enum Upload {
Page(PendingPage),
Data { key: OwnedDataKey, frame: DataFrame },
}
impl Upload {
async fn run(&mut self, pages: &PageStore) -> Result<(), ObjectLogError> {
match self {
Self::Page(page) => {
let raw = std::mem::take(&mut page.raw);
pages.write_immutable(raw, &page.hash, &page.charge).await
}
Self::Data { key, frame } => pages.upload_data(key.as_str(), frame.bytes()).await,
}
}
}
#[derive(Default)]
pub(in crate::native_v2) struct UploadBatch {
pending: Vec<Upload>,
}
impl UploadBatch {
pub fn new() -> Self {
Self::default()
}
pub async fn push(&mut self, pages: &PageStore, upload: Upload) -> Result<(), ObjectLogError> {
self.pending.push(upload);
if self.pending.len() >= UPLOAD_CONCURRENCY {
self.flush(pages).await?;
}
Ok(())
}
pub async fn finish(mut self, pages: &PageStore) -> Result<(), ObjectLogError> {
self.flush(pages).await
}
async fn flush(&mut self, pages: &PageStore) -> Result<(), ObjectLogError> {
let mut batch = std::mem::take(&mut self.pending);
let results = join_all(batch.iter_mut().map(|upload| upload.run(pages))).await;
drop(batch);
results.into_iter().collect()
}
}
async fn join_all<'a, T>(
futures: impl Iterator<Item = impl Future<Output = T> + Send + 'a>,
) -> Vec<T> {
let mut futures: Vec<Pin<Box<dyn Future<Output = T> + Send + 'a>>> =
futures.map(|future| Box::pin(future) as _).collect();
let mut outputs: Vec<Option<T>> = futures.iter().map(|_| None).collect();
std::future::poll_fn(|cx| {
let mut waiting = false;
for (future, output) in futures.iter_mut().zip(outputs.iter_mut()) {
if output.is_none() {
match future.as_mut().poll(cx) {
Poll::Ready(value) => *output = Some(value),
Poll::Pending => waiting = true,
}
}
}
if waiting {
Poll::Pending
} else {
Poll::Ready(())
}
})
.await;
outputs
.into_iter()
.map(|output| output.expect("completed upload"))
.collect()
}