use std::collections::{HashMap, HashSet};
use std::fmt;
use std::io;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use futures_util::{Stream, StreamExt as _};
use multer::Field;
use objectstore_types::headers;
use objectstore_types::metadata::{Compression, Metadata};
use reqwest::header::{CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue};
use reqwest::multipart::Part;
use crate::error::Error;
use crate::put::PutBody;
use crate::{
DeleteBuilder, DeleteResponse, GetBuilder, GetResponse, HeadBuilder, HeadResponse, ObjectKey,
PutBuilder, PutResponse, Session, get, put,
};
const HEADER_BATCH_OPERATION_KEY: &str = "x-sn-batch-operation-key";
const HEADER_BATCH_OPERATION_KIND: &str = "x-sn-batch-operation-kind";
const HEADER_BATCH_OPERATION_INDEX: &str = "x-sn-batch-operation-index";
const HEADER_BATCH_OPERATION_STATUS: &str = "x-sn-batch-operation-status";
const MAX_BATCH_OPS: usize = 1000;
const MAX_BATCH_PART_SIZE: u32 = 1024 * 1024;
const DEFAULT_INDIVIDUAL_CONCURRENCY: usize = 5;
const DEFAULT_BATCH_CONCURRENCY: usize = 3;
const MAX_BATCH_BODY_SIZE: u64 = 100 * 1024 * 1024;
#[derive(Debug)]
pub struct ManyBuilder {
session: Session,
operations: Vec<BatchOperation>,
max_individual_concurrency: Option<usize>,
max_batch_concurrency: Option<usize>,
}
impl Session {
pub fn many(&self) -> ManyBuilder {
ManyBuilder {
session: self.clone(),
operations: vec![],
max_individual_concurrency: None,
max_batch_concurrency: None,
}
}
}
#[derive(Debug)]
#[allow(clippy::large_enum_variant)]
enum BatchOperation {
Get {
key: ObjectKey,
decompress: bool,
accept_encoding: Vec<Compression>,
},
Insert {
key: Option<ObjectKey>,
metadata: Metadata,
body: PutBody,
},
Delete {
key: ObjectKey,
},
Head {
key: ObjectKey,
},
}
impl From<GetBuilder> for BatchOperation {
fn from(value: GetBuilder) -> Self {
let GetBuilder {
key,
decompress,
accept_encoding,
session: _session,
} = value;
BatchOperation::Get {
key,
decompress,
accept_encoding,
}
}
}
impl From<PutBuilder> for BatchOperation {
fn from(value: PutBuilder) -> Self {
let PutBuilder {
key,
metadata,
body,
session: _session,
} = value;
BatchOperation::Insert {
key,
metadata,
body,
}
}
}
impl From<DeleteBuilder> for BatchOperation {
fn from(value: DeleteBuilder) -> Self {
let DeleteBuilder {
key,
session: _session,
} = value;
BatchOperation::Delete { key }
}
}
impl From<HeadBuilder> for BatchOperation {
fn from(value: HeadBuilder) -> Self {
let HeadBuilder {
key,
session: _session,
} = value;
BatchOperation::Head { key }
}
}
impl BatchOperation {
async fn into_part(self) -> crate::Result<Part> {
match self {
BatchOperation::Get { key, .. } => {
let headers = operation_headers("get", Some(&key));
Ok(Part::text("").headers(headers))
}
BatchOperation::Insert {
key,
metadata,
body,
} => {
let mut headers = operation_headers("insert", key.as_deref());
headers.extend(metadata.to_headers("")?);
let body = put::maybe_compress(body, metadata.compression).await?;
Ok(Part::stream(body).headers(headers))
}
BatchOperation::Delete { key } => {
let headers = operation_headers("delete", Some(&key));
Ok(Part::text("").headers(headers))
}
BatchOperation::Head { key } => {
let headers = operation_headers("head", Some(&key));
Ok(Part::text("").headers(headers))
}
}
}
}
fn operation_headers(operation: &str, key: Option<&str>) -> HeaderMap {
let mut headers = HeaderMap::new();
headers.insert(
HeaderName::from_static(HEADER_BATCH_OPERATION_KIND),
HeaderValue::from_str(operation).expect("operation kind is always a valid header value"),
);
if let Some(key) = key {
headers.insert(
HeaderName::from_static(HEADER_BATCH_OPERATION_KEY),
headers::encode_header_value(key),
);
}
headers
}
#[derive(Debug)]
pub enum OperationResult {
Get(ObjectKey, Result<Option<GetResponse>, Error>),
Put(ObjectKey, Result<PutResponse, Error>),
Delete(ObjectKey, Result<DeleteResponse, Error>),
Head(ObjectKey, Result<HeadResponse, Error>),
Error(Error),
}
enum OperationContext {
Get {
key: ObjectKey,
decompress: bool,
accept_encoding: Vec<Compression>,
},
Insert {
key: Option<ObjectKey>,
},
Delete {
key: ObjectKey,
},
Head {
key: ObjectKey,
},
}
impl From<&BatchOperation> for OperationContext {
fn from(op: &BatchOperation) -> Self {
match op {
BatchOperation::Get {
key,
decompress,
accept_encoding,
} => OperationContext::Get {
key: key.clone(),
decompress: *decompress,
accept_encoding: accept_encoding.clone(),
},
BatchOperation::Insert { key, .. } => OperationContext::Insert { key: key.clone() },
BatchOperation::Delete { key } => OperationContext::Delete { key: key.clone() },
BatchOperation::Head { key } => OperationContext::Head { key: key.clone() },
}
}
}
impl OperationContext {
fn key(&self) -> Option<&str> {
match self {
OperationContext::Get { key, .. }
| OperationContext::Delete { key }
| OperationContext::Head { key } => Some(key),
OperationContext::Insert { key } => key.as_deref(),
}
}
}
#[derive(Debug)]
enum Classified {
Batchable(BatchOperation, u64),
Individual(BatchOperation),
Failed(OperationResult),
}
fn error_result(ctx: OperationContext, error: Error) -> OperationResult {
let key = ctx.key().unwrap_or("<unknown>").to_owned();
match ctx {
OperationContext::Get { .. } => OperationResult::Get(key, Err(error)),
OperationContext::Insert { .. } => OperationResult::Put(key, Err(error)),
OperationContext::Delete { .. } => OperationResult::Delete(key, Err(error)),
OperationContext::Head { .. } => OperationResult::Head(key, Err(error)),
}
}
impl OperationResult {
async fn from_field(
field: Field<'_>,
context_map: &HashMap<usize, OperationContext>,
) -> (Option<usize>, Self) {
match Self::try_from_field(field, context_map).await {
Ok((index, result)) => (Some(index), result),
Err(e) => (None, OperationResult::Error(e)),
}
}
async fn try_from_field(
field: Field<'_>,
context_map: &HashMap<usize, OperationContext>,
) -> Result<(usize, Self), Error> {
let mut headers = field.headers().clone();
let index: usize = headers
.remove(HEADER_BATCH_OPERATION_INDEX)
.and_then(|v| v.to_str().ok().and_then(|s| s.parse().ok()))
.ok_or_else(|| {
Error::MalformedResponse(format!(
"missing or invalid {HEADER_BATCH_OPERATION_INDEX} header"
))
})?;
let status: u16 = headers
.remove(HEADER_BATCH_OPERATION_STATUS)
.and_then(|v| {
v.to_str().ok().and_then(|s| {
s.split_once(' ')
.map(|(code, _)| code)
.unwrap_or(s)
.parse()
.ok()
})
})
.ok_or_else(|| {
Error::MalformedResponse(format!(
"missing or invalid {HEADER_BATCH_OPERATION_STATUS} header"
))
})?;
let ctx = context_map.get(&index).ok_or_else(|| {
Error::MalformedResponse(format!(
"response references unknown operation index {index}"
))
})?;
let key = headers
.remove(HEADER_BATCH_OPERATION_KEY)
.and_then(|v| headers::decode_header_value(&v).ok())
.or_else(|| ctx.key().map(str::to_owned));
let body = field.bytes().await?;
let is_error = status >= 400
&& !(matches!(
ctx,
OperationContext::Get { .. } | OperationContext::Head { .. }
) && status == 404);
let key = match key {
Some(key) => key,
None if is_error => "<unknown>".to_owned(),
None => {
return Err(Error::MalformedResponse(format!(
"missing or invalid {HEADER_BATCH_OPERATION_KEY} header"
)));
}
};
if is_error {
let message = String::from_utf8_lossy(&body).into_owned();
let error = Error::OperationFailure { status, message };
return Ok((
index,
match ctx {
OperationContext::Get { .. } => OperationResult::Get(key, Err(error)),
OperationContext::Insert { .. } => OperationResult::Put(key, Err(error)),
OperationContext::Delete { .. } => OperationResult::Delete(key, Err(error)),
OperationContext::Head { .. } => OperationResult::Head(key, Err(error)),
},
));
}
let result = match ctx {
OperationContext::Get {
decompress,
accept_encoding,
..
} => {
if status == 404 {
OperationResult::Get(key, Ok(None))
} else {
let mut metadata = Metadata::from_headers(&headers, "")?;
let stream =
futures_util::stream::once(async move { Ok::<_, io::Error>(body) }).boxed();
let stream =
get::maybe_decompress(stream, &mut metadata, *decompress, accept_encoding);
OperationResult::Get(key, Ok(Some(GetResponse { metadata, stream })))
}
}
OperationContext::Insert { .. } => {
OperationResult::Put(key.clone(), Ok(PutResponse { key }))
}
OperationContext::Delete { .. } => OperationResult::Delete(key, Ok(())),
OperationContext::Head { .. } => {
if status == 404 {
OperationResult::Head(key, Ok(None))
} else {
let metadata = Metadata::from_headers(&headers, "")?;
OperationResult::Head(key, Ok(Some(metadata)))
}
}
};
Ok((index, result))
}
}
pub struct OperationResults(Pin<Box<dyn Stream<Item = OperationResult> + Send>>);
impl fmt::Debug for OperationResults {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("OperationResults([Stream])")
}
}
impl Stream for OperationResults {
type Item = OperationResult;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
self.0.as_mut().poll_next(cx)
}
}
impl OperationResults {
pub async fn error_for_failures(mut self) -> crate::Result<(), impl Iterator<Item = Error>> {
let mut errs = Vec::new();
while let Some(res) = self.next().await {
match res {
OperationResult::Get(_, get) => {
if let Err(e) = get {
errs.push(e);
}
}
OperationResult::Put(_, put) => {
if let Err(e) = put {
errs.push(e);
}
}
OperationResult::Delete(_, delete) => {
if let Err(e) = delete {
errs.push(e);
}
}
OperationResult::Head(_, head) => {
if let Err(e) = head {
errs.push(e);
}
}
OperationResult::Error(error) => errs.push(error),
}
}
if errs.is_empty() {
return Ok(());
}
Err(errs.into_iter())
}
}
async fn send_batch(
session: &Session,
operations: Vec<BatchOperation>,
) -> crate::Result<Vec<OperationResult>> {
let mut context_map: HashMap<usize, OperationContext> = operations
.iter()
.enumerate()
.map(|(idx, op)| (idx, OperationContext::from(op)))
.collect();
let num_operations = operations.len();
let mut form = reqwest::multipart::Form::new();
for op in operations {
let part = op.into_part().await?;
form = form.part("part", part);
}
let request = session.batch_request()?.multipart(form);
let response = request.send().await?.error_for_status()?;
let boundary = response
.headers()
.get(CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.ok_or_else(|| Error::MalformedResponse("missing Content-Type header".to_owned()))
.map(multer::parse_boundary)??;
let byte_stream = response.bytes_stream().map(|r| r.map_err(io::Error::other));
let mut multipart = multer::Multipart::new(byte_stream, boundary);
let mut results = Vec::new();
let mut seen_indices = HashSet::new();
while let Some(field) = multipart.next_field().await? {
let (index, result) = OperationResult::from_field(field, &context_map).await;
if let Some(idx) = index {
seen_indices.insert(idx);
}
results.push(result);
}
for idx in 0..num_operations {
if !seen_indices.contains(&idx) {
let error = Error::MalformedResponse(format!(
"server did not return a response for operation at index {idx}"
));
let result = match context_map.remove(&idx) {
Some(ctx) => error_result(ctx, error),
None => OperationResult::Error(error),
};
results.push(result);
}
}
Ok(results)
}
fn classify_fail(key: Option<ObjectKey>, error: Error) -> Classified {
Classified::Failed(OperationResult::Put(
key.unwrap_or_else(|| "<unknown>".to_owned()),
Err(error),
))
}
async fn classify(op: BatchOperation) -> Classified {
match op {
BatchOperation::Insert {
key,
metadata,
body,
} => {
let size = match &body {
PutBody::Buffer(bytes) => Some(bytes.len() as u64),
PutBody::File(file) => match file.metadata().await {
Ok(meta) => Some(meta.len()),
Err(err) => return classify_fail(key, err.into()),
},
PutBody::Path(path) => match tokio::fs::metadata(path).await {
Ok(meta) => Some(meta.len()),
Err(err) => return classify_fail(key, err.into()),
},
PutBody::Stream(_) => None,
};
let size = match (metadata.compression, size) {
(Some(Compression::Zstd), Some(size)) => {
usize::try_from(size).ok().map(zstd_safe::compress_bound)
}
(None, Some(size)) => usize::try_from(size).ok(),
(_, None) => None,
};
let op = BatchOperation::Insert {
key,
metadata,
body,
};
match size {
Some(s) if s <= MAX_BATCH_PART_SIZE as usize => Classified::Batchable(op, s as u64),
_ => Classified::Individual(op),
}
}
other => Classified::Batchable(other, 0),
}
}
async fn partition(
operations: Vec<BatchOperation>,
) -> (
Vec<(BatchOperation, u64)>,
Vec<BatchOperation>,
Vec<OperationResult>,
) {
let classified = futures_util::future::join_all(operations.into_iter().map(classify)).await;
let mut batchable = Vec::new();
let mut individual = Vec::new();
let mut failed = Vec::new();
for item in classified {
match item {
Classified::Batchable(op, size) => batchable.push((op, size)),
Classified::Individual(op) => individual.push(op),
Classified::Failed(result) => failed.push(result),
}
}
(batchable, individual, failed)
}
async fn execute_individual(op: BatchOperation, session: &Session) -> OperationResult {
match op {
BatchOperation::Get {
key,
decompress,
accept_encoding,
} => {
let get = GetBuilder {
session: session.clone(),
key: key.clone(),
decompress,
accept_encoding,
};
OperationResult::Get(key, get.send().await)
}
BatchOperation::Insert {
key,
metadata,
body,
} => {
let error_key = key.clone().unwrap_or_else(|| "<unknown>".to_owned());
let put = PutBuilder {
session: session.clone(),
metadata,
key,
body,
};
match put.send().await {
Ok(response) => OperationResult::Put(response.key.clone(), Ok(response)),
Err(err) => OperationResult::Put(error_key, Err(err)),
}
}
BatchOperation::Delete { key } => {
let delete = DeleteBuilder {
session: session.clone(),
key: key.clone(),
};
OperationResult::Delete(key, delete.send().await)
}
BatchOperation::Head { key } => {
let head = HeadBuilder {
session: session.clone(),
key: key.clone(),
};
OperationResult::Head(key, head.send().await)
}
}
}
async fn execute_batch(operations: Vec<BatchOperation>, session: &Session) -> Vec<OperationResult> {
let contexts: Vec<_> = operations.iter().map(OperationContext::from).collect();
match send_batch(session, operations).await {
Ok(results) => results,
Err(e) => {
let shared = Arc::new(e);
contexts
.into_iter()
.map(|ctx| error_result(ctx, Error::Batch(shared.clone())))
.collect()
}
}
}
fn iter_batches(ops: Vec<(BatchOperation, u64)>) -> impl Iterator<Item = Vec<BatchOperation>> {
let mut remaining = ops.into_iter().peekable();
std::iter::from_fn(move || {
remaining.peek()?;
let mut batch_size = 0;
let mut batch = Vec::new();
while let Some((_, op_size)) = remaining.peek() {
if batch.len() >= MAX_BATCH_OPS
|| (!batch.is_empty() && batch_size + op_size > MAX_BATCH_BODY_SIZE)
{
break;
}
let (op, op_size) = remaining.next().expect("peeked above");
batch_size += op_size;
batch.push(op);
}
Some(batch)
})
}
impl ManyBuilder {
pub async fn send(self) -> OperationResults {
let session = self.session;
let individual_concurrency = self
.max_individual_concurrency
.unwrap_or(DEFAULT_INDIVIDUAL_CONCURRENCY)
.max(1);
let batch_concurrency = self
.max_batch_concurrency
.unwrap_or(DEFAULT_BATCH_CONCURRENCY)
.max(1);
let (batchable, individual, failed) = partition(self.operations).await;
let individual_results = futures_util::stream::iter(individual)
.map({
let session = session.clone();
move |op| {
let session = session.clone();
async move { execute_individual(op, &session).await }
}
})
.buffer_unordered(individual_concurrency);
let batch_results = futures_util::stream::iter(iter_batches(batchable))
.map(move |chunk| {
let session = session.clone();
async move { execute_batch(chunk, &session).await }
})
.buffer_unordered(batch_concurrency)
.flat_map(futures_util::stream::iter);
let results = futures_util::stream::iter(failed)
.chain(individual_results)
.chain(batch_results);
OperationResults(results.boxed())
}
pub fn max_individual_concurrency(mut self, concurrency: usize) -> Self {
self.max_individual_concurrency = Some(concurrency);
self
}
pub fn max_batch_concurrency(mut self, concurrency: usize) -> Self {
self.max_batch_concurrency = Some(concurrency);
self
}
#[allow(private_bounds)]
pub fn push<B: Into<BatchOperation>>(mut self, builder: B) -> Self {
self.operations.push(builder.into());
self
}
}
#[cfg(test)]
mod tests {
use super::*;
fn op(size: u64) -> (BatchOperation, u64) {
(
BatchOperation::Delete {
key: "k".to_owned(),
},
size,
)
}
fn batch_sizes(batches: &[Vec<BatchOperation>]) -> Vec<usize> {
batches.iter().map(Vec::len).collect()
}
fn batches(ops: Vec<(BatchOperation, u64)>) -> Vec<Vec<BatchOperation>> {
iter_batches(ops).collect()
}
fn put_with_zstd(size: usize) -> BatchOperation {
BatchOperation::Insert {
key: Some("k".to_owned()),
metadata: Metadata {
compression: Some(Compression::Zstd),
..Default::default()
},
body: PutBody::Buffer(vec![0; size].into()),
}
}
#[tokio::test]
async fn zstd_put_at_limit_is_batchable() {
let size = 1_044_496;
let post_compression = zstd_safe::compress_bound(size);
assert!(post_compression == MAX_BATCH_PART_SIZE as usize);
core::assert_matches!(
classify(put_with_zstd(size)).await,
Classified::Batchable(_, s) if s == post_compression as u64
);
}
#[tokio::test]
async fn zstd_put_above_limit_is_individual() {
let size = 1_044_497;
let post_compression = zstd_safe::compress_bound(size);
assert!(post_compression > MAX_BATCH_PART_SIZE as usize);
core::assert_matches!(
classify(put_with_zstd(size)).await,
Classified::Individual(_)
);
}
#[test]
fn iter_batches_empty() {
assert!(batches(vec![]).is_empty());
}
#[test]
fn iter_batches_single_batch_count_limit() {
let ops: Vec<_> = (0..1000).map(|_| op(1)).collect();
assert_eq!(batch_sizes(&batches(ops)), vec![1000]);
}
#[test]
fn iter_batches_splits_on_count_limit() {
let ops: Vec<_> = (0..1001).map(|_| op(1)).collect();
assert_eq!(batch_sizes(&batches(ops)), vec![1000, 1]);
}
#[test]
fn iter_batches_exactly_at_size_limit() {
let ops: Vec<_> = (0..100).map(|_| op(1024 * 1024)).collect();
assert_eq!(batch_sizes(&batches(ops)), vec![100]);
}
#[test]
fn iter_batches_splits_on_size_limit() {
let ops: Vec<_> = (0..101).map(|_| op(1024 * 1024)).collect();
assert_eq!(batch_sizes(&batches(ops)), vec![100, 1]);
}
#[test]
fn iter_batches_size_limit_hits_before_count_limit() {
let op_size = 600 * 1024;
let ops: Vec<_> = (0..200).map(|_| op(op_size)).collect();
let result = batches(ops);
let per_batch = (MAX_BATCH_BODY_SIZE / op_size) as usize;
assert!(result.len() > 1, "expected multiple batches");
for batch in &result[..result.len() - 1] {
assert_eq!(batch.len(), per_batch);
}
}
}