use std::collections::HashMap;
use std::future::Future;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Mutex;
use base64::Engine as _;
use futures::stream::{self, StreamExt as _};
use crate::utils::hash::sha256_hex;
use crate::{ErrorCode, ErrorPayload, GetBlobRet};
pub const DEFAULT_CHUNK_SIZE: u64 = 256 * 1024;
pub const DEFAULT_MAX_RETRIES: usize = 3;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BlobChunkRequest {
pub computer: String,
pub blob_handle: String,
pub chunk_offset: u64,
pub max_chunk_bytes: u64,
}
#[derive(Debug, Clone, Copy)]
pub struct DrainBlobOptions {
pub concurrency: usize,
pub chunk_size: u64,
pub max_retries: usize,
}
impl Default for DrainBlobOptions {
fn default() -> Self {
Self {
concurrency: 1,
chunk_size: DEFAULT_CHUNK_SIZE,
max_retries: DEFAULT_MAX_RETRIES,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum BlobErrorReason {
InvalidHandle,
Forbidden,
Gone,
Range,
Other(String),
}
impl BlobErrorReason {
pub fn parse(reason: &str) -> Self {
match reason {
"invalid_handle" => Self::InvalidHandle,
"forbidden" => Self::Forbidden,
"gone" => Self::Gone,
"range" => Self::Range,
other => Self::Other(other.to_string()),
}
}
pub fn as_str(&self) -> &str {
match self {
Self::InvalidHandle => "invalid_handle",
Self::Forbidden => "forbidden",
Self::Gone => "gone",
Self::Range => "range",
Self::Other(s) => s,
}
}
}
impl std::fmt::Display for BlobErrorReason {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum BlobTransferError {
#[error("blob not accessible (reason: {reason}): {message}")]
NotAccessible {
reason: BlobErrorReason,
message: String,
},
#[error("blob source drift unresolved after {retries} reread(s)")]
MaxRetriesExceeded {
retries: usize,
},
#[error("blob chunk base64 decode failed: {0}")]
Decode(String),
#[error("blob transfer protocol error (code {code}): {message}")]
Protocol {
code: i64,
message: String,
},
}
pub async fn drain_blob<F, Fut>(
call: F,
computer: &str,
blob_handle: &str,
opts: DrainBlobOptions,
) -> Result<(Vec<u8>, String), BlobTransferError>
where
F: Fn(BlobChunkRequest) -> Fut,
Fut: Future<Output = Result<GetBlobRet, ErrorPayload>>,
{
let chunk_size = effective_chunk(opts.chunk_size);
let max_retries = opts.max_retries.max(1);
if opts.concurrency <= 1 {
return drain_serial_async(&call, computer, blob_handle, chunk_size, max_retries).await;
}
match drain_parallel_async(&call, computer, blob_handle, chunk_size, opts.concurrency).await {
Ok(v) => Ok(v),
Err(ParallelErr::Fatal(e)) => Err(e),
Err(ParallelErr::Fallback) => {
drain_serial_async(&call, computer, blob_handle, chunk_size, max_retries).await
}
}
}
pub fn drain_blob_sync<F>(
call: F,
computer: &str,
blob_handle: &str,
opts: DrainBlobOptions,
) -> Result<(Vec<u8>, String), BlobTransferError>
where
F: Fn(BlobChunkRequest) -> Result<GetBlobRet, ErrorPayload> + Sync,
{
let chunk_size = effective_chunk(opts.chunk_size);
let max_retries = opts.max_retries.max(1);
if opts.concurrency <= 1 {
return drain_serial_sync(&call, computer, blob_handle, chunk_size, max_retries);
}
match drain_parallel_sync(&call, computer, blob_handle, chunk_size, opts.concurrency) {
Ok(v) => Ok(v),
Err(ParallelErr::Fatal(e)) => Err(e),
Err(ParallelErr::Fallback) => {
drain_serial_sync(&call, computer, blob_handle, chunk_size, max_retries)
}
}
}
enum SerialErr {
Drift,
Fatal(BlobTransferError),
}
enum ParallelErr {
Fatal(BlobTransferError),
Fallback,
}
enum ChunkErr {
Fatal(BlobTransferError),
Range,
}
async fn drain_serial_async<F, Fut>(
call: &F,
computer: &str,
blob_handle: &str,
chunk_size: u64,
max_retries: usize,
) -> Result<(Vec<u8>, String), BlobTransferError>
where
F: Fn(BlobChunkRequest) -> Fut,
Fut: Future<Output = Result<GetBlobRet, ErrorPayload>>,
{
for _ in 0..max_retries {
match do_serial_drain_async(call, computer, blob_handle, chunk_size).await {
Ok(v) => return Ok(v),
Err(SerialErr::Drift) => continue,
Err(SerialErr::Fatal(e)) => return Err(e),
}
}
Err(BlobTransferError::MaxRetriesExceeded {
retries: max_retries,
})
}
async fn do_serial_drain_async<F, Fut>(
call: &F,
computer: &str,
blob_handle: &str,
chunk_size: u64,
) -> Result<(Vec<u8>, String), SerialErr>
where
F: Fn(BlobChunkRequest) -> Fut,
Fut: Future<Output = Result<GetBlobRet, ErrorPayload>>,
{
let mut state = SerialState::new();
loop {
let req = chunk_request(computer, blob_handle, state.offset, chunk_size);
let ret = call(req)
.await
.map_err(|e| SerialErr::Fatal(classify_fatal(&e)))?;
if state.absorb(ret)? {
break;
}
}
state.finish()
}
fn drain_serial_sync<F>(
call: &F,
computer: &str,
blob_handle: &str,
chunk_size: u64,
max_retries: usize,
) -> Result<(Vec<u8>, String), BlobTransferError>
where
F: Fn(BlobChunkRequest) -> Result<GetBlobRet, ErrorPayload>,
{
for _ in 0..max_retries {
match do_serial_drain_sync(call, computer, blob_handle, chunk_size) {
Ok(v) => return Ok(v),
Err(SerialErr::Drift) => continue,
Err(SerialErr::Fatal(e)) => return Err(e),
}
}
Err(BlobTransferError::MaxRetriesExceeded {
retries: max_retries,
})
}
fn do_serial_drain_sync<F>(
call: &F,
computer: &str,
blob_handle: &str,
chunk_size: u64,
) -> Result<(Vec<u8>, String), SerialErr>
where
F: Fn(BlobChunkRequest) -> Result<GetBlobRet, ErrorPayload>,
{
let mut state = SerialState::new();
loop {
let req = chunk_request(computer, blob_handle, state.offset, chunk_size);
let ret = call(req).map_err(|e| SerialErr::Fatal(classify_fatal(&e)))?;
if state.absorb(ret)? {
break;
}
}
state.finish()
}
struct SerialState {
offset: u64,
acc: Vec<u8>,
first_sha: Option<String>,
first_size: u64,
mime: String,
}
impl SerialState {
fn new() -> Self {
Self {
offset: 0,
acc: Vec::new(),
first_sha: None,
first_size: 0,
mime: String::new(),
}
}
fn absorb(&mut self, ret: GetBlobRet) -> Result<bool, SerialErr> {
match &self.first_sha {
None => {
self.first_sha = Some(ret.sha256.clone());
self.first_size = ret.total_size;
self.mime = ret.mime_type.clone().unwrap_or_default();
}
Some(first) => {
if &ret.sha256 != first || ret.total_size != self.first_size {
return Err(SerialErr::Drift);
}
}
}
let decoded =
b64_decode(&ret.blob).map_err(|e| SerialErr::Fatal(BlobTransferError::Decode(e)))?;
self.offset = ret.chunk_offset + decoded.len() as u64;
self.acc.extend_from_slice(&decoded);
Ok(ret.eof)
}
fn finish(self) -> Result<(Vec<u8>, String), SerialErr> {
let expected = self.first_sha.ok_or(SerialErr::Drift)?;
if sha256_hex(&self.acc) != expected {
return Err(SerialErr::Drift);
}
Ok((self.acc, self.mime))
}
}
async fn drain_parallel_async<F, Fut>(
call: &F,
computer: &str,
blob_handle: &str,
chunk_size: u64,
concurrency: usize,
) -> Result<(Vec<u8>, String), ParallelErr>
where
F: Fn(BlobChunkRequest) -> Fut,
Fut: Future<Output = Result<GetBlobRet, ErrorPayload>>,
{
let first_req = chunk_request(computer, blob_handle, 0, chunk_size);
let first = call(first_req)
.await
.map_err(|e| ParallelErr::Fatal(classify_fatal(&e)))?;
let head = ParallelHead::from_first(first)?;
if let Some(done) = head.single_chunk_result() {
return done;
}
let offsets = head.remaining_offsets(chunk_size);
let mut chunks: HashMap<u64, Vec<u8>> = HashMap::new();
let mut fatal: Option<BlobTransferError> = None;
let mut recoverable = false;
{
let mut stream = stream::iter(offsets.into_iter().map(|off| {
let req = chunk_request(computer, blob_handle, off, chunk_size);
let fut = call(req);
async move { (off, fut.await) }
}))
.buffer_unordered(concurrency);
while let Some((off, res)) = stream.next().await {
match head.absorb_parallel(off, res) {
Ok(Some((off, bytes))) => {
chunks.insert(off, bytes);
}
Ok(None) => recoverable = true,
Err(ChunkErr::Range) => recoverable = true,
Err(ChunkErr::Fatal(e)) => {
fatal = Some(e);
break;
}
}
}
}
if let Some(e) = fatal {
return Err(ParallelErr::Fatal(e));
}
if recoverable {
return Err(ParallelErr::Fallback);
}
head.reassemble(chunks)
}
fn drain_parallel_sync<F>(
call: &F,
computer: &str,
blob_handle: &str,
chunk_size: u64,
concurrency: usize,
) -> Result<(Vec<u8>, String), ParallelErr>
where
F: Fn(BlobChunkRequest) -> Result<GetBlobRet, ErrorPayload> + Sync,
{
let first_req = chunk_request(computer, blob_handle, 0, chunk_size);
let first = call(first_req).map_err(|e| ParallelErr::Fatal(classify_fatal(&e)))?;
let head = ParallelHead::from_first(first)?;
if let Some(done) = head.single_chunk_result() {
return done;
}
let offsets = head.remaining_offsets(chunk_size);
let next = AtomicUsize::new(0);
let stop = AtomicBool::new(false);
let recoverable = AtomicBool::new(false);
let results: Mutex<HashMap<u64, Vec<u8>>> = Mutex::new(HashMap::new());
let fatal: Mutex<Option<BlobTransferError>> = Mutex::new(None);
let workers = concurrency.min(offsets.len()).max(1);
std::thread::scope(|scope| {
for _ in 0..workers {
scope.spawn(|| loop {
if stop.load(Ordering::Relaxed) {
break;
}
let idx = next.fetch_add(1, Ordering::Relaxed);
let Some(&off) = offsets.get(idx) else { break };
let req = chunk_request(computer, blob_handle, off, chunk_size);
match head.absorb_parallel(off, call(req)) {
Ok(Some((off, bytes))) => {
results.lock().unwrap().insert(off, bytes);
}
Ok(None) => {
recoverable.store(true, Ordering::Relaxed);
}
Err(ChunkErr::Range) => {
recoverable.store(true, Ordering::Relaxed);
}
Err(ChunkErr::Fatal(e)) => {
let mut guard = fatal.lock().unwrap();
if guard.is_none() {
*guard = Some(e);
}
stop.store(true, Ordering::Relaxed);
break;
}
}
});
}
});
if let Some(e) = fatal.into_inner().unwrap() {
return Err(ParallelErr::Fatal(e));
}
if recoverable.load(Ordering::Relaxed) {
return Err(ParallelErr::Fallback);
}
head.reassemble(results.into_inner().unwrap())
}
struct ParallelHead {
total_size: u64,
expected_sha: String,
mime: String,
first_bytes: Vec<u8>,
first_len: u64,
}
impl ParallelHead {
fn from_first(first: GetBlobRet) -> Result<Self, ParallelErr> {
let first_bytes = b64_decode(&first.blob)
.map_err(|e| ParallelErr::Fatal(BlobTransferError::Decode(e)))?;
let first_len = first_bytes.len() as u64;
Ok(Self {
total_size: first.total_size,
expected_sha: first.sha256,
mime: first.mime_type.unwrap_or_default(),
first_bytes,
first_len,
})
}
fn single_chunk_result(&self) -> Option<Result<(Vec<u8>, String), ParallelErr>> {
if self.total_size == 0 || self.first_len >= self.total_size {
if sha256_hex(&self.first_bytes) != self.expected_sha {
return Some(Err(ParallelErr::Fallback));
}
return Some(Ok((self.first_bytes.clone(), self.mime.clone())));
}
None
}
fn remaining_offsets(&self, chunk_size: u64) -> Vec<u64> {
(self.first_len..self.total_size)
.step_by(chunk_size as usize)
.collect()
}
fn absorb_parallel(
&self,
off: u64,
res: Result<GetBlobRet, ErrorPayload>,
) -> Result<Option<(u64, Vec<u8>)>, ChunkErr> {
match res {
Err(e) => Err(classify_chunk(&e)),
Ok(ret) => {
if ret.sha256 != self.expected_sha || ret.total_size != self.total_size {
return Ok(None); }
let bytes = b64_decode(&ret.blob)
.map_err(|e| ChunkErr::Fatal(BlobTransferError::Decode(e)))?;
Ok(Some((off, bytes)))
}
}
}
fn reassemble(
&self,
mut chunks: HashMap<u64, Vec<u8>>,
) -> Result<(Vec<u8>, String), ParallelErr> {
chunks.insert(0, self.first_bytes.clone());
let mut keys: Vec<u64> = chunks.keys().copied().collect();
keys.sort_unstable();
let mut full = Vec::with_capacity(self.total_size as usize);
for k in keys {
full.extend_from_slice(&chunks[&k]);
}
if sha256_hex(&full) != self.expected_sha {
return Err(ParallelErr::Fallback);
}
Ok((full, self.mime.clone()))
}
}
fn effective_chunk(chunk_size: u64) -> u64 {
if chunk_size == 0 {
DEFAULT_CHUNK_SIZE
} else {
chunk_size
}
}
fn chunk_request(computer: &str, blob_handle: &str, offset: u64, max: u64) -> BlobChunkRequest {
BlobChunkRequest {
computer: computer.to_owned(),
blob_handle: blob_handle.to_owned(),
chunk_offset: offset,
max_chunk_bytes: max,
}
}
fn classify_fatal(err: &ErrorPayload) -> BlobTransferError {
if err.code == i64::from(ErrorCode::BlobNotAccessible.code()) {
BlobTransferError::NotAccessible {
reason: extract_reason(err),
message: err.message.clone(),
}
} else {
BlobTransferError::Protocol {
code: err.code,
message: err.message.clone(),
}
}
}
fn classify_chunk(err: &ErrorPayload) -> ChunkErr {
if err.code == i64::from(ErrorCode::BlobNotAccessible.code()) {
let reason = extract_reason(err);
if matches!(reason, BlobErrorReason::Range) {
return ChunkErr::Range;
}
ChunkErr::Fatal(BlobTransferError::NotAccessible {
reason,
message: err.message.clone(),
})
} else {
ChunkErr::Fatal(BlobTransferError::Protocol {
code: err.code,
message: err.message.clone(),
})
}
}
fn extract_reason(err: &ErrorPayload) -> BlobErrorReason {
let raw = err
.details
.as_ref()
.and_then(|d| d.get("reason"))
.and_then(|v| v.as_str())
.unwrap_or("invalid_handle");
BlobErrorReason::parse(raw)
}
fn b64_decode(s: &str) -> Result<Vec<u8>, String> {
base64::engine::general_purpose::STANDARD
.decode(s)
.map_err(|e| e.to_string())
}
#[cfg(test)]
#[allow(clippy::result_large_err)]
mod tests {
use super::*;
use std::sync::Arc;
fn b64_encode(bytes: &[u8]) -> String {
base64::engine::general_purpose::STANDARD.encode(bytes)
}
fn serve(data: &[u8], sha: &str, offset: u64, max: u64) -> GetBlobRet {
let start = (offset as usize).min(data.len());
let end = (start + max as usize).min(data.len());
let chunk = &data[start..end];
GetBlobRet {
blob_handle: "h".to_string(),
mime_type: Some("application/octet-stream".to_string()),
total_size: data.len() as u64,
sha256: sha.to_string(),
chunk_offset: offset,
eof: end >= data.len(),
blob: b64_encode(chunk),
req_id: None,
}
}
fn err_4018(reason: &str) -> ErrorPayload {
ErrorPayload::new(
i64::from(ErrorCode::BlobNotAccessible.code()),
"blob not accessible",
)
.with_detail("reason", reason)
}
fn sample(len: usize) -> Vec<u8> {
(0..len).map(|i| (i % 251) as u8).collect()
}
#[tokio::test]
async fn serial_async_multi_chunk_reassembles() {
let data = Arc::new(sample(1000));
let sha = sha256_hex(&data);
let call = {
let data = data.clone();
let sha = sha.clone();
move |req: BlobChunkRequest| {
let data = data.clone();
let sha = sha.clone();
async move {
Ok::<_, ErrorPayload>(serve(&data, &sha, req.chunk_offset, req.max_chunk_bytes))
}
}
};
let opts = DrainBlobOptions {
concurrency: 1,
chunk_size: 128,
max_retries: 3,
};
let (bytes, mime) = drain_blob(call, "c", "h", opts).await.unwrap();
assert_eq!(bytes, *data);
assert_eq!(mime, "application/octet-stream");
}
#[tokio::test]
async fn serial_async_empty_blob() {
let data: Vec<u8> = Vec::new();
let sha = sha256_hex(&data);
let call = move |req: BlobChunkRequest| {
let sha = sha.clone();
async move { Ok::<_, ErrorPayload>(serve(&[], &sha, req.chunk_offset, req.max_chunk_bytes)) }
};
let (bytes, _mime) = drain_blob(call, "c", "h", DrainBlobOptions::default())
.await
.unwrap();
assert!(bytes.is_empty());
}
#[tokio::test]
async fn serial_async_sha_mismatch_exhausts_retries() {
let data = Arc::new(sample(300));
let bogus = "0".repeat(64);
let call = {
let data = data.clone();
move |req: BlobChunkRequest| {
let data = data.clone();
let bogus = bogus.clone();
async move {
Ok::<_, ErrorPayload>(serve(
&data,
&bogus,
req.chunk_offset,
req.max_chunk_bytes,
))
}
}
};
let opts = DrainBlobOptions {
concurrency: 1,
chunk_size: 128,
max_retries: 2,
};
let err = drain_blob(call, "c", "h", opts).await.unwrap_err();
assert_eq!(err, BlobTransferError::MaxRetriesExceeded { retries: 2 });
}
#[tokio::test]
async fn serial_async_total_size_drift_then_recovers() {
let old = Arc::new(sample(400));
let new = Arc::new(sample(640));
let old_sha = sha256_hex(&old);
let new_sha = sha256_hex(&new);
let started_reread = Arc::new(AtomicBool::new(false));
let seen_offsets = Arc::new(Mutex::new(Vec::<u64>::new()));
let call = {
let old = old.clone();
let new = new.clone();
let old_sha = old_sha.clone();
let new_sha = new_sha.clone();
let started_reread = started_reread.clone();
let seen_offsets = seen_offsets.clone();
move |req: BlobChunkRequest| {
let old = old.clone();
let new = new.clone();
let old_sha = old_sha.clone();
let new_sha = new_sha.clone();
let started_reread = started_reread.clone();
let seen_offsets = seen_offsets.clone();
async move {
if req.chunk_offset == 0 && seen_offsets.lock().unwrap().contains(&0) {
started_reread.store(true, Ordering::SeqCst);
}
seen_offsets.lock().unwrap().push(req.chunk_offset);
if started_reread.load(Ordering::SeqCst) {
Ok::<_, ErrorPayload>(serve(
&new,
&new_sha,
req.chunk_offset,
req.max_chunk_bytes,
))
} else if req.chunk_offset < 256 {
Ok(serve(&old, &old_sha, req.chunk_offset, req.max_chunk_bytes))
} else {
Ok(serve(&new, &new_sha, req.chunk_offset, req.max_chunk_bytes))
}
}
}
};
let opts = DrainBlobOptions {
concurrency: 1,
chunk_size: 128,
max_retries: 3,
};
let (bytes, _mime) = drain_blob(call, "c", "h", opts).await.unwrap();
assert!(started_reread.load(Ordering::SeqCst), "应触发从 0 重读");
assert_eq!(bytes, *new);
}
#[tokio::test]
async fn serial_async_4018_invalid_handle_is_fatal() {
let call =
|_req: BlobChunkRequest| async { Err::<GetBlobRet, _>(err_4018("invalid_handle")) };
let err = drain_blob(call, "c", "h", DrainBlobOptions::default())
.await
.unwrap_err();
assert!(matches!(
err,
BlobTransferError::NotAccessible {
reason: BlobErrorReason::InvalidHandle,
..
}
));
}
#[tokio::test]
async fn serial_async_4018_range_is_fatal_in_serial() {
let call = |_req: BlobChunkRequest| async { Err::<GetBlobRet, _>(err_4018("range")) };
let err = drain_blob(call, "c", "h", DrainBlobOptions::default())
.await
.unwrap_err();
assert!(matches!(
err,
BlobTransferError::NotAccessible {
reason: BlobErrorReason::Range,
..
}
));
}
#[tokio::test]
async fn serial_async_other_protocol_code_surfaces() {
let call = |_req: BlobChunkRequest| async {
Err::<GetBlobRet, _>(ErrorPayload::new(4014, "nope"))
};
let err = drain_blob(call, "c", "h", DrainBlobOptions::default())
.await
.unwrap_err();
assert_eq!(
err,
BlobTransferError::Protocol {
code: 4014,
message: "nope".to_string()
}
);
}
#[tokio::test]
async fn parallel_async_multi_chunk_reassembles() {
let data = Arc::new(sample(5000));
let sha = sha256_hex(&data);
let call = {
let data = data.clone();
let sha = sha.clone();
move |req: BlobChunkRequest| {
let data = data.clone();
let sha = sha.clone();
async move {
Ok::<_, ErrorPayload>(serve(&data, &sha, req.chunk_offset, req.max_chunk_bytes))
}
}
};
let opts = DrainBlobOptions {
concurrency: 4,
chunk_size: 256,
max_retries: 3,
};
let (bytes, _mime) = drain_blob(call, "c", "h", opts).await.unwrap();
assert_eq!(bytes, *data);
}
#[tokio::test]
async fn parallel_async_range_falls_back_to_serial() {
let data = Arc::new(sample(2000));
let sha = sha256_hex(&data);
let parallel_seen = Arc::new(AtomicBool::new(false));
let call = {
let data = data.clone();
let sha = sha.clone();
let parallel_seen = parallel_seen.clone();
move |req: BlobChunkRequest| {
let data = data.clone();
let sha = sha.clone();
let parallel_seen = parallel_seen.clone();
async move {
if req.chunk_offset > 0 && !parallel_seen.load(Ordering::SeqCst) {
parallel_seen.store(true, Ordering::SeqCst);
return Err::<GetBlobRet, _>(err_4018("range"));
}
Ok(serve(&data, &sha, req.chunk_offset, req.max_chunk_bytes))
}
}
};
let opts = DrainBlobOptions {
concurrency: 4,
chunk_size: 256,
max_retries: 3,
};
let (bytes, _mime) = drain_blob(call, "c", "h", opts).await.unwrap();
assert!(parallel_seen.load(Ordering::SeqCst), "应曾触发并行 range");
assert_eq!(bytes, *data);
}
#[tokio::test]
async fn parallel_async_invalid_handle_is_fatal_no_fallback() {
let data = Arc::new(sample(2000));
let sha = sha256_hex(&data);
let call = {
let data = data.clone();
let sha = sha.clone();
move |req: BlobChunkRequest| {
let data = data.clone();
let sha = sha.clone();
async move {
if req.chunk_offset > 0 {
return Err::<GetBlobRet, _>(err_4018("forbidden"));
}
Ok(serve(&data, &sha, req.chunk_offset, req.max_chunk_bytes))
}
}
};
let opts = DrainBlobOptions {
concurrency: 4,
chunk_size: 256,
max_retries: 3,
};
let err = drain_blob(call, "c", "h", opts).await.unwrap_err();
assert!(matches!(
err,
BlobTransferError::NotAccessible {
reason: BlobErrorReason::Forbidden,
..
}
));
}
#[test]
fn serial_sync_multi_chunk_reassembles() {
let data = sample(1000);
let sha = sha256_hex(&data);
let call = |req: BlobChunkRequest| {
Ok::<_, ErrorPayload>(serve(&data, &sha, req.chunk_offset, req.max_chunk_bytes))
};
let opts = DrainBlobOptions {
concurrency: 1,
chunk_size: 128,
max_retries: 3,
};
let (bytes, _mime) = drain_blob_sync(call, "c", "h", opts).unwrap();
assert_eq!(bytes, data);
}
#[test]
fn parallel_sync_multi_chunk_reassembles() {
let data = sample(5000);
let sha = sha256_hex(&data);
let call = |req: BlobChunkRequest| {
Ok::<_, ErrorPayload>(serve(&data, &sha, req.chunk_offset, req.max_chunk_bytes))
};
let opts = DrainBlobOptions {
concurrency: 4,
chunk_size: 256,
max_retries: 3,
};
let (bytes, _mime) = drain_blob_sync(call, "c", "h", opts).unwrap();
assert_eq!(bytes, data);
}
#[test]
fn parallel_sync_range_falls_back_to_serial() {
let data = sample(2000);
let sha = sha256_hex(&data);
let parallel_seen = AtomicBool::new(false);
let call = |req: BlobChunkRequest| {
if req.chunk_offset > 0 && !parallel_seen.load(Ordering::SeqCst) {
parallel_seen.store(true, Ordering::SeqCst);
return Err::<GetBlobRet, _>(err_4018("range"));
}
Ok(serve(&data, &sha, req.chunk_offset, req.max_chunk_bytes))
};
let opts = DrainBlobOptions {
concurrency: 4,
chunk_size: 256,
max_retries: 3,
};
let (bytes, _mime) = drain_blob_sync(call, "c", "h", opts).unwrap();
assert!(parallel_seen.load(Ordering::SeqCst));
assert_eq!(bytes, data);
}
#[test]
fn sync_4018_gone_is_fatal() {
let call = |_req: BlobChunkRequest| Err::<GetBlobRet, _>(err_4018("gone"));
let err = drain_blob_sync(call, "c", "h", DrainBlobOptions::default()).unwrap_err();
assert!(matches!(
err,
BlobTransferError::NotAccessible {
reason: BlobErrorReason::Gone,
..
}
));
}
#[test]
fn reason_parse_round_trip() {
for r in ["invalid_handle", "forbidden", "gone", "range"] {
assert_eq!(BlobErrorReason::parse(r).as_str(), r);
}
assert_eq!(
BlobErrorReason::parse("future_reason"),
BlobErrorReason::Other("future_reason".to_string())
);
}
#[tokio::test]
async fn parallel_async_drift_falls_back_to_serial() {
let data = Arc::new(sample(2000));
let sha = sha256_hex(&data);
let wrong = "0".repeat(64);
let injected = Arc::new(AtomicBool::new(false));
let call = {
let data = data.clone();
let sha = sha.clone();
let wrong = wrong.clone();
let injected = injected.clone();
move |req: BlobChunkRequest| {
let data = data.clone();
let sha = sha.clone();
let wrong = wrong.clone();
let injected = injected.clone();
async move {
let use_sha = if req.chunk_offset > 0 && !injected.swap(true, Ordering::SeqCst)
{
&wrong
} else {
&sha
};
Ok::<_, ErrorPayload>(serve(
&data,
use_sha,
req.chunk_offset,
req.max_chunk_bytes,
))
}
}
};
let opts = DrainBlobOptions {
concurrency: 4,
chunk_size: 256,
max_retries: 3,
};
let (bytes, _mime) = drain_blob(call, "c", "h", opts).await.unwrap();
assert!(injected.load(Ordering::SeqCst), "应曾注入并行漂移");
assert_eq!(bytes, *data);
}
#[tokio::test]
async fn parallel_async_fatal_beats_recoverable() {
let data = Arc::new(sample(2000));
let sha = sha256_hex(&data);
let call = {
let data = data.clone();
let sha = sha.clone();
move |req: BlobChunkRequest| {
let data = data.clone();
let sha = sha.clone();
async move {
match req.chunk_offset {
256 => Err::<GetBlobRet, _>(err_4018("range")),
512 => Err(err_4018("forbidden")),
off => Ok(serve(&data, &sha, off, req.max_chunk_bytes)),
}
}
}
};
let opts = DrainBlobOptions {
concurrency: 4,
chunk_size: 256,
max_retries: 3,
};
let err = drain_blob(call, "c", "h", opts).await.unwrap_err();
assert!(
matches!(
err,
BlobTransferError::NotAccessible {
reason: BlobErrorReason::Forbidden,
..
}
),
"fatal 必须胜出,得到的却是 {err:?}"
);
}
#[test]
fn parallel_sync_fatal_beats_recoverable() {
let data = sample(2000);
let sha = sha256_hex(&data);
let call = |req: BlobChunkRequest| match req.chunk_offset {
256 => Err::<GetBlobRet, _>(err_4018("range")),
512 => Err(err_4018("forbidden")),
off => Ok(serve(&data, &sha, off, req.max_chunk_bytes)),
};
let opts = DrainBlobOptions {
concurrency: 4,
chunk_size: 256,
max_retries: 3,
};
let err = drain_blob_sync(call, "c", "h", opts).unwrap_err();
assert!(
matches!(
err,
BlobTransferError::NotAccessible {
reason: BlobErrorReason::Forbidden,
..
}
),
"sync fatal 必须胜出,得到的却是 {err:?}"
);
}
#[test]
fn parallel_sync_drift_falls_back_to_serial() {
let data = sample(2000);
let sha = sha256_hex(&data);
let wrong = "0".repeat(64);
let injected = AtomicBool::new(false);
let call = |req: BlobChunkRequest| {
let use_sha = if req.chunk_offset > 0 && !injected.swap(true, Ordering::SeqCst) {
&wrong
} else {
&sha
};
Ok::<_, ErrorPayload>(serve(&data, use_sha, req.chunk_offset, req.max_chunk_bytes))
};
let opts = DrainBlobOptions {
concurrency: 4,
chunk_size: 256,
max_retries: 3,
};
let (bytes, _mime) = drain_blob_sync(call, "c", "h", opts).unwrap();
assert!(injected.load(Ordering::SeqCst));
assert_eq!(bytes, data);
}
#[tokio::test]
async fn parallel_async_single_chunk_sha_mismatch_falls_back() {
let data = Arc::new(sample(100)); let sha = sha256_hex(&data);
let wrong = "0".repeat(64);
let count = Arc::new(AtomicUsize::new(0));
let call = {
let data = data.clone();
let sha = sha.clone();
let wrong = wrong.clone();
let count = count.clone();
move |req: BlobChunkRequest| {
let data = data.clone();
let sha = sha.clone();
let wrong = wrong.clone();
let count = count.clone();
async move {
let use_sha = if count.fetch_add(1, Ordering::SeqCst) == 0 {
&wrong
} else {
&sha
};
Ok::<_, ErrorPayload>(serve(
&data,
use_sha,
req.chunk_offset,
req.max_chunk_bytes,
))
}
}
};
let opts = DrainBlobOptions {
concurrency: 4,
chunk_size: 256,
max_retries: 3,
};
let (bytes, _mime) = drain_blob(call, "c", "h", opts).await.unwrap();
assert_eq!(bytes, *data);
}
#[tokio::test]
async fn parallel_async_reassemble_sha_mismatch_exhausts() {
let data = Arc::new(sample(2000));
let bogus = "0".repeat(64);
let call = {
let data = data.clone();
let bogus = bogus.clone();
move |req: BlobChunkRequest| {
let data = data.clone();
let bogus = bogus.clone();
async move {
Ok::<_, ErrorPayload>(serve(
&data,
&bogus,
req.chunk_offset,
req.max_chunk_bytes,
))
}
}
};
let opts = DrainBlobOptions {
concurrency: 4,
chunk_size: 256,
max_retries: 2,
};
let err = drain_blob(call, "c", "h", opts).await.unwrap_err();
assert_eq!(err, BlobTransferError::MaxRetriesExceeded { retries: 2 });
}
#[tokio::test]
async fn serial_async_decode_error() {
let call = |_req: BlobChunkRequest| async {
Ok::<_, ErrorPayload>(GetBlobRet {
blob_handle: "h".to_string(),
mime_type: Some("application/octet-stream".to_string()),
total_size: 8,
sha256: "deadbeef".to_string(),
chunk_offset: 0,
eof: true,
blob: "@@@not-base64@@@".to_string(),
req_id: None,
})
};
let err = drain_blob(call, "c", "h", DrainBlobOptions::default())
.await
.unwrap_err();
assert!(matches!(err, BlobTransferError::Decode(_)), "got {err:?}");
}
#[test]
fn serial_sync_sha_mismatch_exhausts_retries() {
let data = sample(300);
let bogus = "0".repeat(64);
let call = |req: BlobChunkRequest| {
Ok::<_, ErrorPayload>(serve(&data, &bogus, req.chunk_offset, req.max_chunk_bytes))
};
let opts = DrainBlobOptions {
concurrency: 1,
chunk_size: 128,
max_retries: 2,
};
let err = drain_blob_sync(call, "c", "h", opts).unwrap_err();
assert_eq!(err, BlobTransferError::MaxRetriesExceeded { retries: 2 });
}
#[tokio::test]
async fn serial_async_zero_retries_still_attempts_once() {
let data = Arc::new(sample(300));
let sha = sha256_hex(&data);
let call = {
let data = data.clone();
let sha = sha.clone();
move |req: BlobChunkRequest| {
let data = data.clone();
let sha = sha.clone();
async move {
Ok::<_, ErrorPayload>(serve(&data, &sha, req.chunk_offset, req.max_chunk_bytes))
}
}
};
let opts = DrainBlobOptions {
concurrency: 1,
chunk_size: 128,
max_retries: 0,
};
let (bytes, _mime) = drain_blob(call, "c", "h", opts).await.unwrap();
assert_eq!(bytes, *data);
}
}