use std::collections::HashMap;
use std::ops::Range;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, Weak};
use bytes::Bytes;
use tokio::sync::Notify;
use tokio_util::sync::CancellationToken;
use crate::{
error::SdxError,
session::DownloadSessionInner,
stream::{DownloadStream, RunState, UnorderedDownloadStream},
upload::{UploadFileInfo, UploadReport, UploadSession, UploadStreamHandle},
};
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum XetTaskState {
Queued,
InProgress,
Completed,
Failed(String),
Cancelled,
}
static NEXT_GROUP_ID: AtomicU64 = AtomicU64::new(0);
#[derive(Clone)]
pub struct XetStreamGroup {
inner: Arc<XetStreamGroupInner>,
}
struct XetStreamGroupInner {
session: Arc<DownloadSessionInner>,
id: u64,
token: CancellationToken,
aborted: AtomicBool,
active: Mutex<HashMap<u64, Weak<StreamRegistration>>>,
next_id: AtomicU64,
}
struct StreamRegistration {
id: u64,
run_state: Arc<RunState>,
start_signal: Option<Arc<Notify>>,
started: AtomicBool,
finished: AtomicBool,
explicitly_cancelled: AtomicBool,
observed_failure: Mutex<Option<String>>,
}
impl StreamRegistration {
fn new(id: u64, stream: &DownloadStream) -> Arc<Self> {
Arc::new(Self {
id,
run_state: stream.run_state(),
start_signal: stream.pending_start_signal(),
started: AtomicBool::new(false),
finished: AtomicBool::new(false),
explicitly_cancelled: AtomicBool::new(false),
observed_failure: Mutex::new(None),
})
}
fn new_unordered(id: u64, stream: &UnorderedDownloadStream) -> Arc<Self> {
Arc::new(Self {
id,
run_state: stream.run_state(),
start_signal: stream.pending_start_signal(),
started: AtomicBool::new(false),
finished: AtomicBool::new(false),
explicitly_cancelled: AtomicBool::new(false),
observed_failure: Mutex::new(None),
})
}
fn cancel(&self) {
self.explicitly_cancelled.store(true, Ordering::Relaxed);
self.run_state.cancel();
if let Some(signal) = &self.start_signal {
signal.notify_one();
}
}
fn observe_result<T>(&self, result: &Result<Option<T>, SdxError>) {
if let Err(error) = result {
self.observed_failure
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.get_or_insert_with(|| error.to_string());
}
if result.as_ref().is_ok_and(Option::is_none) {
self.finished.store(true, Ordering::Relaxed);
}
}
fn state(&self, group_aborted: bool) -> XetTaskState {
if group_aborted || self.explicitly_cancelled.load(Ordering::Relaxed) {
return XetTaskState::Cancelled;
}
let observed = self
.observed_failure
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.clone();
if let Some(message) = observed.or_else(|| self.run_state.error_message()) {
return XetTaskState::Failed(message);
}
if self.run_state.is_cancelled() {
return XetTaskState::Cancelled;
}
if self.finished.load(Ordering::Relaxed) {
return XetTaskState::Completed;
}
if self.started.load(Ordering::Relaxed) {
return XetTaskState::InProgress;
}
XetTaskState::Queued
}
}
impl XetStreamGroupInner {
fn register(&self, registration: &Arc<StreamRegistration>) {
if self.aborted.load(Ordering::Relaxed) {
registration.cancel();
}
let mut active = self
.active
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
active.insert(registration.id, Arc::downgrade(registration));
}
fn unregister(&self, id: u64) {
let mut active = self
.active
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
active.remove(&id);
}
}
impl XetStreamGroup {
pub(crate) fn new(session: Arc<DownloadSessionInner>) -> Self {
Self {
inner: Arc::new(XetStreamGroupInner {
session,
id: NEXT_GROUP_ID.fetch_add(1, Ordering::Relaxed),
token: CancellationToken::new(),
aborted: AtomicBool::new(false),
active: Mutex::new(HashMap::new()),
next_id: AtomicU64::new(0),
}),
}
}
#[must_use]
pub fn id(&self) -> u64 {
self.inner.id
}
#[must_use]
pub fn is_aborted(&self) -> bool {
self.inner.aborted.load(Ordering::Relaxed)
}
pub async fn download_stream(
&self,
file_id: &str,
range: Option<Range<u64>>,
) -> Result<GroupedDownloadStream, SdxError> {
let id = self.inner.next_id.fetch_add(1, Ordering::Relaxed);
let stream_token = self.inner.token.child_token();
let mut reconstructor = self
.inner
.session
.build_reconstructor(file_id, range)
.await?;
reconstructor = reconstructor.with_cancellation_token(stream_token.clone());
let stream = reconstructor.reconstruct_to_stream();
let registration = StreamRegistration::new(id, &stream);
self.register(®istration);
Ok(GroupedDownloadStream::new(
stream,
id,
registration,
Arc::downgrade(&self.inner),
))
}
pub async fn download_unordered_stream(
&self,
file_id: &str,
range: Option<Range<u64>>,
) -> Result<GroupedUnorderedDownloadStream, SdxError> {
let id = self.inner.next_id.fetch_add(1, Ordering::Relaxed);
let stream_token = self.inner.token.child_token();
let mut reconstructor = self
.inner
.session
.build_reconstructor(file_id, range)
.await?;
reconstructor = reconstructor.with_cancellation_token(stream_token.clone());
let stream = reconstructor.reconstruct_to_unordered_stream();
let registration = StreamRegistration::new_unordered(id, &stream);
self.register(®istration);
Ok(GroupedUnorderedDownloadStream::new(
stream,
id,
registration,
Arc::downgrade(&self.inner),
))
}
pub fn abort(&self) {
self.inner.aborted.store(true, Ordering::Relaxed);
self.inner.token.cancel();
let active = self
.inner
.active
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
for weak in active.values() {
if let Some(registration) = weak.upgrade() {
registration.cancel();
}
}
}
#[must_use]
pub fn status(&self) -> Vec<(u64, XetTaskState)> {
let aborted = self.inner.aborted.load(Ordering::Relaxed);
let active = self
.inner
.active
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let mut states = Vec::with_capacity(active.len());
for (id, weak) in active.iter() {
let Some(registration) = weak.upgrade() else {
continue;
};
states.push((*id, registration.state(aborted)));
}
states.sort_by_key(|(id, _)| *id);
states
}
#[must_use]
pub fn active_stream_count(&self) -> usize {
let active = self
.inner
.active
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
active
.values()
.filter(|weak| weak.upgrade().is_some())
.count()
}
fn register(&self, registration: &Arc<StreamRegistration>) {
self.inner.register(registration);
}
}
pub struct GroupedDownloadStream {
inner: DownloadStream,
id: u64,
registration: Arc<StreamRegistration>,
group: Weak<XetStreamGroupInner>,
}
impl GroupedDownloadStream {
const fn new(
inner: DownloadStream,
id: u64,
registration: Arc<StreamRegistration>,
group: Weak<XetStreamGroupInner>,
) -> Self {
Self {
inner,
id,
registration,
group,
}
}
#[must_use]
pub const fn task_id(&self) -> u64 {
self.id
}
pub fn start(&mut self) {
self.registration.started.store(true, Ordering::Relaxed);
self.inner.start();
}
pub async fn next(&mut self) -> Result<Option<Bytes>, SdxError> {
self.registration.started.store(true, Ordering::Relaxed);
let result = self.inner.next().await;
self.registration.observe_result(&result);
result
}
#[cfg(not(target_family = "wasm"))]
pub fn blocking_next(&mut self) -> Result<Option<Bytes>, SdxError> {
self.registration.started.store(true, Ordering::Relaxed);
let result = self.inner.blocking_next();
self.registration.observe_result(&result);
result
}
pub fn cancel(&mut self) {
self.registration.cancel();
self.inner.cancel();
}
}
impl Drop for GroupedDownloadStream {
fn drop(&mut self) {
if let Some(group) = self.group.upgrade() {
group.unregister(self.id);
}
}
}
pub struct GroupedUnorderedDownloadStream {
inner: UnorderedDownloadStream,
id: u64,
registration: Arc<StreamRegistration>,
group: Weak<XetStreamGroupInner>,
}
impl GroupedUnorderedDownloadStream {
const fn new(
inner: UnorderedDownloadStream,
id: u64,
registration: Arc<StreamRegistration>,
group: Weak<XetStreamGroupInner>,
) -> Self {
Self {
inner,
id,
registration,
group,
}
}
#[must_use]
pub const fn task_id(&self) -> u64 {
self.id
}
pub fn start(&mut self) {
self.registration.started.store(true, Ordering::Relaxed);
self.inner.start();
}
pub async fn next(&mut self) -> Result<Option<(u64, Bytes)>, SdxError> {
self.registration.started.store(true, Ordering::Relaxed);
let result = self.inner.next().await;
self.registration.observe_result(&result);
result
}
#[cfg(not(target_family = "wasm"))]
pub fn blocking_next(&mut self) -> Result<Option<(u64, Bytes)>, SdxError> {
self.registration.started.store(true, Ordering::Relaxed);
let result = self.inner.blocking_next();
self.registration.observe_result(&result);
result
}
pub fn cancel(&mut self) {
self.registration.cancel();
self.inner.cancel();
}
}
impl Drop for GroupedUnorderedDownloadStream {
fn drop(&mut self) {
if let Some(group) = self.group.upgrade() {
group.unregister(self.id);
}
}
}
#[derive(Clone)]
pub struct XetUploadCommit {
inner: Arc<XetUploadCommitInner>,
}
struct XetUploadCommitInner {
session: UploadSession,
id: u64,
token: CancellationToken,
aborted: AtomicBool,
active: Mutex<HashMap<u64, Weak<UploadRegistration>>>,
next_id: AtomicU64,
#[cfg(test)]
registration_gate: Option<Arc<std::sync::Barrier>>,
}
struct UploadRegistration {
handle: UploadStreamHandle,
}
impl UploadRegistration {
fn state(&self, group_aborted: bool) -> XetTaskState {
let flags = self.handle.status_flags();
if group_aborted || flags.aborted {
return XetTaskState::Cancelled;
}
if let Some(message) = flags.error {
return XetTaskState::Failed(message);
}
if flags.finished {
return XetTaskState::Completed;
}
if flags.started {
return XetTaskState::InProgress;
}
XetTaskState::Queued
}
}
impl XetUploadCommit {
pub(crate) fn new(session: &Arc<DownloadSessionInner>) -> Result<Self, SdxError> {
let upload_session = UploadSession::new(session)?;
Ok(Self {
inner: Arc::new(XetUploadCommitInner {
session: upload_session,
id: NEXT_GROUP_ID.fetch_add(1, Ordering::Relaxed),
token: CancellationToken::new(),
aborted: AtomicBool::new(false),
active: Mutex::new(HashMap::new()),
next_id: AtomicU64::new(0),
#[cfg(test)]
registration_gate: None,
}),
})
}
#[must_use]
pub fn id(&self) -> u64 {
self.inner.id
}
#[must_use]
pub fn is_aborted(&self) -> bool {
self.inner.aborted.load(Ordering::Relaxed)
}
pub fn upload_stream(&self) -> Result<GroupedUploadStream, SdxError> {
let id = self.inner.next_id.fetch_add(1, Ordering::Relaxed);
#[cfg(test)]
if let Some(gate) = &self.inner.registration_gate {
gate.wait();
gate.wait();
}
let mut active = self
.inner
.active
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
if self.inner.aborted.load(Ordering::Relaxed) {
return Err(SdxError::UploadSession(
"upload group aborted; cannot start new streams".to_owned(),
));
}
let handle = self.inner.session.upload_stream_handle_with_id(id);
let registration = Arc::new(UploadRegistration {
handle: handle.clone(),
});
active.insert(id, Arc::downgrade(®istration));
Ok(GroupedUploadStream::new(
handle,
id,
registration,
Arc::downgrade(&self.inner),
))
}
pub async fn commit(&self) -> Result<UploadReport, SdxError> {
if self.inner.aborted.load(Ordering::Relaxed) {
return Err(SdxError::UploadSession(
"upload group aborted; cannot commit".to_owned(),
));
}
self.inner.session.finalize().await
}
pub fn abort(&self) {
self.inner.aborted.store(true, Ordering::Relaxed);
self.inner.token.cancel();
let active = self
.inner
.active
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
for weak in active.values() {
if let Some(registration) = weak.upgrade() {
registration.handle.abort();
}
}
}
#[must_use]
pub fn status(&self) -> Vec<(u64, XetTaskState)> {
let aborted = self.inner.aborted.load(Ordering::Relaxed);
let active = self
.inner
.active
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let mut states = Vec::with_capacity(active.len());
for (id, weak) in active.iter() {
let Some(registration) = weak.upgrade() else {
continue;
};
states.push((*id, registration.state(aborted)));
}
states.sort_by_key(|(id, _)| *id);
states
}
#[must_use]
pub fn active_upload_count(&self) -> usize {
let active = self
.inner
.active
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
active
.values()
.filter(|weak| weak.upgrade().is_some())
.count()
}
}
impl XetUploadCommitInner {
fn unregister(&self, id: u64) {
let mut active = self
.active
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
active.remove(&id);
}
}
pub struct GroupedUploadStream {
handle: UploadStreamHandle,
id: u64,
registration: Arc<UploadRegistration>,
group: Weak<XetUploadCommitInner>,
}
impl GroupedUploadStream {
const fn new(
handle: UploadStreamHandle,
id: u64,
registration: Arc<UploadRegistration>,
group: Weak<XetUploadCommitInner>,
) -> Self {
Self {
handle,
id,
registration,
group,
}
}
#[must_use]
pub const fn task_id(&self) -> u64 {
self.id
}
pub async fn write(&self, data: impl Into<Bytes>) -> Result<(), SdxError> {
self.handle.write(data).await
}
#[cfg(not(target_family = "wasm"))]
pub fn write_blocking(&self, data: impl Into<Bytes>) -> Result<(), SdxError> {
self.handle.write_blocking(data)
}
pub async fn finish(&self) -> Result<UploadFileInfo, SdxError> {
self.handle.finish().await
}
#[must_use]
pub fn try_finish(&self) -> Option<UploadFileInfo> {
self.handle.try_finish()
}
#[must_use]
pub fn status(&self) -> XetTaskState {
let aborted = self
.group
.upgrade()
.is_some_and(|group| group.aborted.load(Ordering::Relaxed));
self.registration.state(aborted)
}
pub fn cancel(&self) {
self.handle.abort();
}
}
impl Drop for GroupedUploadStream {
fn drop(&mut self) {
self.handle.abort();
if let Some(group) = self.group.upgrade() {
group.unregister(self.id);
}
}
}
#[cfg(test)]
mod tests {
use std::time::Duration;
use serde_json::json;
use tokio::time::timeout;
use wiremock::{
Mock, MockServer, ResponseTemplate,
matchers::{header, method, path},
};
use super::*;
use crate::{Auth, RepositoryId, XetClientBuilder};
const FILE_ID: &str = "0000000000000000000000000000000000000000000000000000000000000000";
const XORB_HASH: &str = "1111111111111111111111111111111111111111111111111111111111111111";
const READ_TOKEN: &str = "read-token";
async fn upload_controls_inside_runtime() {
let auth = Auth::new(
"http://127.0.0.1:1",
RepositoryId {
provider: "github".to_owned(),
owner: "team".to_owned(),
repo: "assets".to_owned(),
revision: "main".to_owned(),
},
)
.unwrap()
.with_api_key("local-test".to_owned());
let client = XetClientBuilder::new()
.endpoint("xet://127.0.0.1:1/github/team/assets/main")
.auth(auth)
.build()
.unwrap();
let direct = client.upload_session().unwrap().upload_stream_handle();
direct.abort();
assert!(direct.write(Bytes::from_static(b"late")).await.is_err());
assert!(direct.finish().await.is_err());
let group = client.new_upload_group().unwrap();
let first = group.upload_stream().unwrap();
let second = group.upload_stream().unwrap();
assert_eq!(first.status(), XetTaskState::Queued);
assert_eq!(group.status().len(), 2);
first.cancel();
assert_eq!(first.status(), XetTaskState::Cancelled);
assert_eq!(second.status(), XetTaskState::Queued);
drop(first);
assert_eq!(group.active_upload_count(), 1);
group.abort();
assert_eq!(second.status(), XetTaskState::Cancelled);
assert!(group.upload_stream().is_err());
drop(second);
assert_eq!(group.active_upload_count(), 0);
assert!(group.status().is_empty());
}
#[tokio::test]
async fn upload_controls_are_safe_in_current_thread_runtime() {
upload_controls_inside_runtime().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn upload_controls_are_safe_in_multithread_runtime() {
upload_controls_inside_runtime().await;
}
async fn upload_client(server: &MockServer, shard_delay: Duration) -> crate::XetClient {
Mock::given(method("GET"))
.and(path("/api/github/team/assets/xet-write-token/main"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"casUrl": server.uri(), "exp": 4_000_000_000u64, "accessToken": "write-token",
})))
.mount(server)
.await;
Mock::given(method("POST"))
.and(wiremock::matchers::path_regex(r"/v1/xorbs/default/.*"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"was_inserted": true})))
.mount(server)
.await;
Mock::given(method("POST"))
.and(path("/v1/shards"))
.respond_with(
ResponseTemplate::new(200)
.set_delay(shard_delay)
.set_body_json(json!({"result": 1})),
)
.mount(server)
.await;
client(server).await
}
#[tokio::test]
async fn retained_upload_status_survives_parent_drop() {
let server = MockServer::start().await;
let client = upload_client(&server, Duration::ZERO).await;
let group = client.new_upload_group().unwrap();
let handle = group.upload_stream().unwrap();
let group_clone = group.clone();
drop(group);
assert_eq!(handle.status(), XetTaskState::Queued);
drop(group_clone);
assert_eq!(handle.status(), XetTaskState::Queued);
handle
.write(Bytes::from_static(b"ordinary payload"))
.await
.unwrap();
assert_eq!(handle.status(), XetTaskState::InProgress);
let info = handle.finish().await.unwrap();
assert_eq!(info.total_bytes, 16);
assert_eq!(handle.try_finish(), Some(info));
assert_eq!(handle.status(), XetTaskState::Completed);
handle.cancel();
assert_eq!(handle.status(), XetTaskState::Cancelled);
assert!(handle.write(Bytes::new()).await.is_err());
assert!(handle.finish().await.is_err());
let aborted_group = client.new_upload_group().unwrap();
let aborted = aborted_group.upload_stream().unwrap();
aborted_group.abort();
drop(aborted_group);
assert_eq!(aborted.status(), XetTaskState::Cancelled);
assert!(aborted.write(Bytes::new()).await.is_err());
assert!(aborted.finish().await.is_err());
tokio::task::unconstrained(async { drop(server) }).await;
}
#[tokio::test]
async fn failed_upload_status_survives_parent_drop() {
let server = MockServer::start().await;
let client = upload_client(&server, Duration::ZERO).await;
Mock::given(method("GET"))
.and(wiremock::matchers::path_regex(
r"/v1/chunks/default-merkledb/.*",
))
.respond_with(
ResponseTemplate::new(400).set_body_json(json!({"error": "owned failure"})),
)
.mount(&server)
.await;
let group = client.new_upload_group().unwrap();
let handle = group.upload_stream().unwrap();
handle
.write(Bytes::from_static(b"ordinary payload"))
.await
.unwrap();
assert!(matches!(
handle.finish().await,
Err(SdxError::Transfer(crate::error::TransferError::BadRequest(
_
)))
));
let failed = handle.status();
assert!(matches!(failed, XetTaskState::Failed(_)));
drop(group);
assert_eq!(handle.status(), failed);
handle.cancel();
assert_eq!(handle.status(), XetTaskState::Cancelled);
tokio::task::unconstrained(async { drop(server) }).await;
}
#[tokio::test]
async fn upload_constructor_resuming_after_abort_cannot_register() {
let server = MockServer::start().await;
let client = client(&server).await;
let mut group = client.new_upload_group().unwrap();
let gate = Arc::new(std::sync::Barrier::new(2));
Arc::get_mut(&mut group.inner).unwrap().registration_gate = Some(gate.clone());
let constructing = group.clone();
let task = std::thread::spawn(move || constructing.upload_stream());
gate.wait();
group.abort();
gate.wait();
assert!(
matches!(task.join().unwrap(), Err(SdxError::UploadSession(message))
if message == "upload group aborted; cannot start new streams")
);
assert!(group.is_aborted());
assert_eq!(group.active_upload_count(), 0);
assert!(group.status().is_empty());
assert!(server.received_requests().await.unwrap().is_empty());
}
#[tokio::test]
async fn pre_aborted_upload_commit_withholds_completed_and_empty_files() {
for payload in [Vec::new(), b"ordinary payload".to_vec()] {
for aborted in [false, true] {
let server = MockServer::start().await;
let client = upload_client(&server, Duration::ZERO).await;
let group = client.new_upload_group().unwrap();
let handle = group.upload_stream().unwrap();
handle.write(payload.clone()).await.unwrap();
let info = handle.finish().await.unwrap();
if aborted {
group.abort();
assert!(
matches!(group.commit().await, Err(SdxError::UploadSession(message)) if message == "upload group aborted; cannot commit")
);
assert_eq!(handle.status(), XetTaskState::Cancelled);
assert!(group.upload_stream().is_err());
} else {
let report = group.commit().await.unwrap();
assert_eq!(report.files, vec![info]);
assert_eq!(report.shard_posts, 1);
}
let requests = server.received_requests().await.unwrap();
assert_eq!(
requests
.iter()
.filter(|request| request.method.as_str() == "POST"
&& request.url.path() == "/v1/shards")
.count(),
usize::from(!aborted)
);
}
}
}
#[tokio::test]
async fn pre_aborted_empty_group_cannot_commit_healthy_empty_group_can() {
let server = MockServer::start().await;
let client = upload_client(&server, Duration::ZERO).await;
let healthy = client.new_upload_group().unwrap();
let report = healthy.commit().await.unwrap();
assert!(report.files.is_empty());
assert_eq!(report.shard_posts, 0);
let aborted = client.new_upload_group().unwrap();
aborted.abort();
assert!(
matches!(aborted.commit().await, Err(SdxError::UploadSession(message)) if message == "upload group aborted; cannot commit")
);
assert!(server.received_requests().await.unwrap().is_empty());
}
#[tokio::test]
async fn abort_does_not_roll_back_an_upload_commit_already_posting() {
let server = MockServer::start().await;
let client = upload_client(&server, Duration::from_millis(100)).await;
let group = client.new_upload_group().unwrap();
let handle = group.upload_stream().unwrap();
let info = handle.finish().await.unwrap();
let committing_group = group.clone();
let committing = tokio::spawn(async move { committing_group.commit().await });
timeout(Duration::from_secs(5), async {
loop {
if server
.received_requests()
.await
.unwrap()
.iter()
.any(|request| request.url.path() == "/v1/shards")
{
break;
}
tokio::task::yield_now().await;
}
})
.await
.unwrap();
group.abort();
let report = committing.await.unwrap().unwrap();
assert_eq!(report.files, vec![info]);
assert_eq!(report.shard_posts, 1);
}
fn serialize_payload(chunks: &[&[u8]]) -> Vec<u8> {
use xet_core_structures::xorb_object::{CompressionScheme, serialize_chunk};
let mut payload = Vec::new();
for chunk in chunks {
serialize_chunk(chunk, &mut payload, CompressionScheme::None).unwrap();
}
payload
}
fn v2_response_body(
offset: u64,
terms: serde_json::Value,
xorbs: serde_json::Value,
) -> serde_json::Value {
let mut body = serde_json::Map::new();
body.insert(
"offset_into_first_range".to_owned(),
serde_json::Value::from(offset),
);
body.insert("terms".to_owned(), terms);
body.insert("xorbs".to_owned(), xorbs);
serde_json::Value::Object(body)
}
fn test_limits() -> crate::StreamLimits {
crate::StreamLimits {
min_reconstruction_fetch_size: 4096,
max_reconstruction_fetch_size: 4096,
min_prefetch_buffer: 8192,
..crate::StreamLimits::default()
}
}
async fn client(server: &MockServer) -> crate::XetClient {
Mock::given(method("GET"))
.and(path("/api/github/team/assets/xet-read-token/main"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"casUrl": server.uri(),
"exp": 4_000_000_000u64,
"accessToken": READ_TOKEN,
})))
.mount(server)
.await;
let auth = Auth::new(
&server.uri(),
RepositoryId {
provider: "github".to_owned(),
owner: "team".to_owned(),
repo: "assets".to_owned(),
revision: "main".to_owned(),
},
)
.unwrap()
.with_api_key("bootstrap".to_owned())
.with_subject("user".to_owned());
let port = server.uri().split(':').next_back().unwrap().to_owned();
XetClientBuilder::new()
.endpoint(format!("xet://127.0.0.1:{port}/github/team/assets/main"))
.auth(auth)
.with_stream_limits(test_limits())
.build()
.unwrap()
}
async fn mocks(server: &MockServer, delay: Option<Duration>) {
let payload = serialize_payload(&[&[7u8; 64], &[9u8; 64]]);
let end = payload.len().checked_sub(1).expect("nonempty xorb fixture");
Mock::given(method("GET"))
.and(path(format!("/v2/reconstructions/{FILE_ID}")))
.and(header("authorization", format!("Bearer {READ_TOKEN}")))
.and(header("range", "bytes=0-4095"))
.respond_with(ResponseTemplate::new(200).set_body_json(v2_response_body(
0,
json!([
{"hash": XORB_HASH, "unpacked_length": 64, "range": {"start": 0, "end": 1}},
{"hash": XORB_HASH, "unpacked_length": 64, "range": {"start": 1, "end": 2}}
]),
json!({
XORB_HASH: [{
"url": format!("{}/transfer/xorb/default/{XORB_HASH}", server.uri()),
"ranges": [
{"chunks": {"start": 0, "end": 2}, "bytes": {"start": 0, "end": end}}
]
}]
}),
)))
.mount(server)
.await;
Mock::given(method("GET"))
.and(path(format!("/v2/reconstructions/{FILE_ID}")))
.and(header("authorization", format!("Bearer {READ_TOKEN}")))
.respond_with(ResponseTemplate::new(416).set_body_json(json!({"error": "past eof"})))
.mount(server)
.await;
let mut template = ResponseTemplate::new(206)
.insert_header("Content-Range", format!("bytes 0-{end}/{}", payload.len()))
.set_body_raw(payload, "application/octet-stream");
if let Some(delay) = delay {
template = template.set_delay(delay);
}
Mock::given(method("GET"))
.and(path(format!("/transfer/xorb/default/{XORB_HASH}")))
.and(header("authorization", format!("Bearer {READ_TOKEN}")))
.and(header("range", format!("bytes=0-{end}")))
.respond_with(template)
.mount(server)
.await;
}
async fn drain(stream: &mut GroupedDownloadStream) -> Vec<u8> {
let mut out = Vec::new();
while let Some(chunk) = stream.next().await.unwrap() {
out.extend_from_slice(&chunk);
}
out
}
#[tokio::test]
async fn status_transitions_queued_to_in_progress_to_completed() {
let server = MockServer::start().await;
mocks(&server, None).await;
let group = client(&server).await.new_download_stream_group();
let mut stream = group.download_stream(FILE_ID, None).await.unwrap();
let id = stream.task_id();
assert_eq!(group.active_stream_count(), 1);
assert!(group.status().contains(&(id, XetTaskState::Queued)));
let chunk = stream.next().await.unwrap().unwrap();
assert_eq!(chunk.as_ref(), vec![7u8; 64]);
assert!(group.status().contains(&(id, XetTaskState::InProgress)));
let out = drain(&mut stream).await;
assert_eq!(out, vec![9u8; 64]);
assert!(group.status().contains(&(id, XetTaskState::Completed)));
}
#[tokio::test]
async fn pending_reconstruction_block_is_in_progress_until_observed_eof() {
for unordered in [false, true] {
let server = MockServer::start().await;
let payload = serialize_payload(&[&[7u8; 64]]);
let end = payload.len().checked_sub(1).expect("nonempty fixture");
let body = v2_response_body(
0,
serde_json::Value::Array(vec![
json!({
"hash": XORB_HASH, "unpacked_length": 64,
"range": {"start": 0, "end": 1}
});
64
]),
json!({XORB_HASH: [{
"url": format!("{}/transfer/xorb/default/{XORB_HASH}", server.uri()),
"ranges": [{"chunks": {"start": 0, "end": 1},
"bytes": {"start": 0, "end": end}}]
}]}),
);
let entered = Arc::new(Notify::new());
let release = Arc::new(Notify::new());
let entered_handler = entered.clone();
let release_handler = release.clone();
let second_body = body.clone();
let app = axum::Router::new().route(
"/second",
axum::routing::get(move || {
let entered = entered_handler.clone();
let release = release_handler.clone();
let body = second_body.clone();
async move {
entered.notify_one();
release.notified().await;
axum::Json(body)
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let gate_server = tokio::spawn(async move { axum::serve(listener, app).await });
Mock::given(method("GET"))
.and(path(format!("/v2/reconstructions/{FILE_ID}")))
.and(header("range", "bytes=0-4095"))
.respond_with(ResponseTemplate::new(200).set_body_json(body))
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/v2/reconstructions/{FILE_ID}")))
.and(header("range", "bytes=4096-8191"))
.respond_with(
ResponseTemplate::new(307)
.insert_header("Location", format!("http://{address}/second")),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/v2/reconstructions/{FILE_ID}")))
.respond_with(ResponseTemplate::new(416))
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/transfer/xorb/default/{XORB_HASH}")))
.respond_with(
ResponseTemplate::new(206)
.insert_header("Content-Range", format!("bytes 0-{end}/{}", payload.len()))
.set_body_raw(payload, "application/octet-stream"),
)
.mount(&server)
.await;
let group = client(&server).await.new_download_stream_group();
let mut out = Vec::new();
if unordered {
let mut stream = group
.download_unordered_stream(FILE_ID, None)
.await
.unwrap();
for _ in 0..64 {
let (_, chunk) = stream.next().await.unwrap().unwrap();
out.extend_from_slice(&chunk);
}
timeout(Duration::from_secs(3), entered.notified())
.await
.unwrap();
assert_eq!(out, vec![7u8; 4096]);
assert!(
group
.status()
.contains(&(stream.task_id(), XetTaskState::InProgress))
);
release.notify_one();
while let Some((_, chunk)) = stream.next().await.unwrap() {
out.extend_from_slice(&chunk);
}
assert!(
group
.status()
.contains(&(stream.task_id(), XetTaskState::Completed))
);
} else {
let mut stream = group.download_stream(FILE_ID, None).await.unwrap();
for _ in 0..64 {
out.extend_from_slice(&stream.next().await.unwrap().unwrap());
}
timeout(Duration::from_secs(3), entered.notified())
.await
.unwrap();
assert_eq!(out, vec![7u8; 4096]);
assert!(
group
.status()
.contains(&(stream.task_id(), XetTaskState::InProgress))
);
release.notify_one();
while let Some(chunk) = stream.next().await.unwrap() {
out.extend_from_slice(&chunk);
}
assert!(
group
.status()
.contains(&(stream.task_id(), XetTaskState::Completed))
);
}
assert_eq!(out, vec![7u8; 8192]);
assert!(group.status().is_empty());
gate_server.abort();
assert!(gate_server.await.unwrap_err().is_cancelled());
tokio::task::unconstrained(async { drop(server) }).await;
}
}
#[tokio::test]
async fn blocking_download_wrappers_complete_only_after_eof() {
for unordered in [false, true] {
let server = MockServer::start().await;
mocks(&server, None).await;
let group = client(&server).await.new_download_stream_group();
if unordered {
let mut stream = group
.download_unordered_stream(FILE_ID, None)
.await
.unwrap();
let group_probe = group.clone();
tokio::task::spawn_blocking(move || {
let id = stream.task_id();
let mut out = Vec::new();
for _ in 0..2 {
let (_, chunk) = stream.blocking_next().unwrap().unwrap();
out.extend_from_slice(&chunk);
}
assert!(
group_probe
.status()
.contains(&(id, XetTaskState::InProgress))
);
assert!(stream.blocking_next().unwrap().is_none());
assert!(
group_probe
.status()
.contains(&(id, XetTaskState::Completed))
);
out.sort_unstable();
assert_eq!(out, [vec![7u8; 64], vec![9u8; 64]].concat());
})
.await
.unwrap();
} else {
let mut stream = group.download_stream(FILE_ID, None).await.unwrap();
let group_probe = group.clone();
tokio::task::spawn_blocking(move || {
let id = stream.task_id();
let mut out = Vec::new();
for _ in 0..2 {
out.extend_from_slice(&stream.blocking_next().unwrap().unwrap());
}
assert!(
group_probe
.status()
.contains(&(id, XetTaskState::InProgress))
);
assert!(stream.blocking_next().unwrap().is_none());
assert!(
group_probe
.status()
.contains(&(id, XetTaskState::Completed))
);
assert_eq!(out, [vec![7u8; 64], vec![9u8; 64]].concat());
})
.await
.unwrap();
}
assert!(group.status().is_empty());
tokio::task::unconstrained(async { drop(server) }).await;
}
}
#[tokio::test]
async fn download_failure_status_survives_typed_error_consumption_and_eof() {
for unordered in [false, true] {
for blocking in [false, true] {
let server = MockServer::start().await;
let group = client(&server).await.new_download_stream_group();
Mock::given(method("GET"))
.and(wiremock::matchers::path_regex(r".*/reconstructions/.*"))
.respond_with(
ResponseTemplate::new(404)
.set_body_json(json!({"error": "owned reconstruction failure"})),
)
.mount(&server)
.await;
if unordered {
let mut stream = group
.download_unordered_stream(FILE_ID, None)
.await
.unwrap();
let id = stream.task_id();
stream.start();
timeout(Duration::from_secs(5), async {
while stream.registration.run_state.error_message().is_none() {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
assert!(group.status().contains(&(
id,
XetTaskState::Failed(
"not found (404): owned reconstruction failure".to_owned()
)
)));
let result = if blocking {
let (returned, result) = tokio::task::spawn_blocking(move || {
let result = stream.blocking_next();
(stream, result)
})
.await
.unwrap();
stream = returned;
result
} else {
stream.next().await
};
assert!(
matches!(result, Err(SdxError::Transfer(crate::error::TransferError::NotFound(message)))
if message == "owned reconstruction failure")
);
assert!(stream.next().await.unwrap().is_none());
assert!(group.status().contains(&(
id,
XetTaskState::Failed(
"not found (404): owned reconstruction failure".to_owned()
)
)));
stream.cancel();
assert!(group.status().contains(&(id, XetTaskState::Cancelled)));
} else {
let mut stream = group.download_stream(FILE_ID, None).await.unwrap();
let id = stream.task_id();
stream.start();
timeout(Duration::from_secs(5), async {
while stream.registration.run_state.error_message().is_none() {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
assert!(group.status().contains(&(
id,
XetTaskState::Failed(
"not found (404): owned reconstruction failure".to_owned()
)
)));
let result = if blocking {
let (returned, result) = tokio::task::spawn_blocking(move || {
let result = stream.blocking_next();
(stream, result)
})
.await
.unwrap();
stream = returned;
result
} else {
stream.next().await
};
assert!(
matches!(result, Err(SdxError::Transfer(crate::error::TransferError::NotFound(message)))
if message == "owned reconstruction failure")
);
assert!(stream.next().await.unwrap().is_none());
assert!(group.status().contains(&(
id,
XetTaskState::Failed(
"not found (404): owned reconstruction failure".to_owned()
)
)));
stream.cancel();
assert!(group.status().contains(&(id, XetTaskState::Cancelled)));
}
}
}
}
#[tokio::test]
async fn group_abort_prevents_a_new_read_after_a_delivered_chunk() {
for unordered in [false, true] {
let server = MockServer::start().await;
mocks(&server, None).await;
let group = client(&server).await.new_download_stream_group();
if unordered {
let mut stream = group
.download_unordered_stream(FILE_ID, None)
.await
.unwrap();
let id = stream.task_id();
assert_eq!(stream.next().await.unwrap().unwrap().1.len(), 64);
group.abort();
assert!(stream.next().await.unwrap().is_none());
assert!(group.status().contains(&(id, XetTaskState::Cancelled)));
} else {
let mut stream = group.download_stream(FILE_ID, None).await.unwrap();
let id = stream.task_id();
assert_eq!(stream.next().await.unwrap().unwrap().len(), 64);
group.abort();
assert!(stream.next().await.unwrap().is_none());
assert!(group.status().contains(&(id, XetTaskState::Cancelled)));
}
}
}
#[tokio::test]
async fn abort_mid_stream_stops_promptly() {
let server = MockServer::start().await;
mocks(&server, Some(Duration::from_secs(30))).await;
let group = client(&server).await.new_download_stream_group();
let mut stream = group.download_stream(FILE_ID, None).await.unwrap();
let id = stream.task_id();
let abort = {
let group = group.clone();
tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(100)).await;
group.abort();
})
};
let next = timeout(Duration::from_secs(5), stream.next())
.await
.expect("abort must stop the stream promptly")
.unwrap();
assert!(next.is_none());
abort.await.unwrap();
assert!(group.is_aborted());
assert!(group.status().contains(&(id, XetTaskState::Cancelled)));
}
#[tokio::test]
async fn abort_cancels_queued_streams_too() {
let server = MockServer::start().await;
mocks(&server, Some(Duration::from_secs(30))).await;
let group = client(&server).await.new_download_stream_group();
let mut queued = group.download_stream(FILE_ID, None).await.unwrap();
let queued_id = queued.task_id();
let mut running = group.download_stream(FILE_ID, None).await.unwrap();
let running_id = running.task_id();
group.abort();
let next = timeout(Duration::from_secs(5), running.next())
.await
.expect("abort must stop the running stream")
.unwrap();
assert!(next.is_none());
let next = timeout(Duration::from_secs(5), queued.next())
.await
.expect("abort must stop the queued stream")
.unwrap();
assert!(next.is_none());
let status = group.status();
assert!(status.contains(&(queued_id, XetTaskState::Cancelled)));
assert!(status.contains(&(running_id, XetTaskState::Cancelled)));
}
#[tokio::test]
async fn drop_unregisters_stream_from_group() {
let server = MockServer::start().await;
mocks(&server, None).await;
let group = client(&server).await.new_download_stream_group();
let stream = group.download_stream(FILE_ID, None).await.unwrap();
assert_eq!(group.active_stream_count(), 1);
drop(stream);
assert_eq!(group.active_stream_count(), 0);
assert!(group.status().is_empty());
}
#[tokio::test]
async fn two_concurrent_streams_are_byte_identical() {
let server = MockServer::start().await;
mocks(&server, None).await;
let group = client(&server).await.new_download_stream_group();
let expected = [vec![7u8; 64], vec![9u8; 64]].concat();
let mut a = group.download_stream(FILE_ID, None).await.unwrap();
let mut b = group
.download_unordered_stream(FILE_ID, None)
.await
.unwrap();
let task_a = tokio::spawn(async move { drain(&mut a).await });
let task_b = tokio::spawn(async move {
let mut pieces = Vec::new();
while let Some((offset, chunk)) = b.next().await.unwrap() {
pieces.push((offset, chunk));
}
pieces.sort_by_key(|(offset, _)| *offset);
let mut out = Vec::new();
for (_, chunk) in pieces {
out.extend_from_slice(&chunk);
}
out
});
let out_a = task_a.await.unwrap();
let out_b = task_b.await.unwrap();
assert_eq!(out_a, expected);
assert_eq!(out_b, expected);
assert_eq!(group.active_stream_count(), 0);
}
#[tokio::test]
async fn stream_cancel_does_not_cancel_siblings() {
let server = MockServer::start().await;
mocks(&server, None).await;
let group = client(&server).await.new_download_stream_group();
let expected = [vec![7u8; 64], vec![9u8; 64]].concat();
let mut a = group.download_stream(FILE_ID, None).await.unwrap();
let mut b = group.download_stream(FILE_ID, None).await.unwrap();
a.cancel();
assert!(a.next().await.unwrap().is_none());
let out = drain(&mut b).await;
assert_eq!(out, expected);
}
#[test]
fn xet_task_state_is_comparable() {
assert_eq!(XetTaskState::Queued, XetTaskState::Queued);
assert_ne!(XetTaskState::Queued, XetTaskState::InProgress);
assert_ne!(
XetTaskState::Failed("x".to_owned()),
XetTaskState::Failed("y".to_owned())
);
}
}