1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
//! Download sessions for the sdx CAS read path (M2a / M2b1).
//!
//! [`DownloadSession`] downloads a file (or a byte range of a file) by its
//! 64-hex `file_id` — the library core is file_id-addressed; path resolution
//! arrives with the §2.5 server metadata endpoints in M5
//! (`docs/SDX_PLAN.md` §4.3). Downloads are sequential and unbuffered-to-disk:
//! the reconstructed bytes are assembled in memory and written to `dest`.
//!
//! M2b1 adds the pull-based streaming surface (`download_stream`,
//! `download_unordered_stream`, `download_to_writer`, `download_bytes`),
//! mirroring upstream `xet-data`'s `FileReconstructor` (`docs/SDX_PLAN.md`
//! §4.4.1).
use std::{
ops::{Range, RangeInclusive},
path::Path,
sync::{
Arc,
atomic::{AtomicU64, Ordering},
},
};
use bytes::Bytes;
use crate::{
auth::TokenService,
cache::ChunkCache,
error::SdxError,
hash::parse_xet_hash_hex,
reconstruction,
retry::{RetryContext, RetryMarkers, RetryPolicy, RetryScope},
stream::{
BufferSemaphore, DownloadStream, FileReconstructor, StreamContext, StreamLimits,
UnorderedDownloadStream,
},
transfer::{ByteRange, TransferClient},
};
/// Shared state between a [`DownloadSession`] and its owning [`crate::XetClient`].
pub(crate) struct DownloadSessionInner {
pub(crate) transfer: TransferClient,
pub(crate) tokens: TokenService,
pub(crate) api_base: String,
pub(crate) buffer_semaphore: Arc<BufferSemaphore>,
pub(crate) active_downloads: Arc<AtomicU64>,
pub(crate) download_permits: Arc<tokio::sync::Semaphore>,
pub(crate) limits: StreamLimits,
/// Optional on-disk chunk cache (M2b2), shared by all streams.
pub(crate) chunk_cache: Option<Arc<ChunkCache>>,
/// Count of ranged xorb transfer requests (for observability/E2E).
pub(crate) xorb_fetch_count: Arc<AtomicU64>,
/// CDC target chunk size for uploads (M3b), default 64 KiB.
pub(crate) upload_chunk_size: usize,
/// Fixed upload connection-permit count (M4), default 2.
pub(crate) upload_concurrency: usize,
/// Retry policy (M4) applied to all CAS requests.
pub(crate) retry_policy: RetryPolicy,
/// Repository identity from the `xet://` endpoint (M5b metadata routes).
pub(crate) repository: crate::auth::RepositoryId,
}
impl DownloadSessionInner {
/// Builds a [`RetryContext`] for a read-scoped CAS request using this
/// session's policy, token service, and session id.
pub(crate) fn retry_context(&self) -> RetryContext {
RetryContext {
policy: self.retry_policy.clone(),
tokens: Some(self.tokens.clone()),
scope: RetryScope::Read,
markers: RetryMarkers::reconstruction(),
}
}
/// Builds the streaming pipeline context shared by this session, resolving
/// the process-global blocking runtime lazily.
///
/// # Errors
///
/// Returns [`SdxError`] when the dedicated blocking runtime cannot be
/// started.
pub(crate) fn stream_context(&self) -> Result<StreamContext, SdxError> {
Ok(StreamContext {
transfer: self.transfer.clone(),
api_base: self.api_base.clone(),
buffer_semaphore: self.buffer_semaphore.clone(),
active_downloads: self.active_downloads.clone(),
download_permits: self.download_permits.clone(),
limits: self.limits.clone(),
chunk_cache: self.chunk_cache.clone(),
xorb_fetch_count: self.xorb_fetch_count.clone(),
tokens: self.tokens.clone(),
retry_policy: self.retry_policy.clone(),
#[cfg(not(target_family = "wasm"))]
blocking_runtime: crate::stream::global_blocking_runtime()?,
})
}
/// Builds a [`FileReconstructor`] for `file_id`, validating the identifier
/// and resolving the read token.
///
/// # Errors
///
/// Returns [`SdxError`] when `file_id` is malformed, `range` is inverted
/// (`start > end`), token issuance fails, or the blocking runtime cannot
/// be resolved.
pub(crate) async fn build_reconstructor(
&self,
file_id: &str,
range: Option<Range<u64>>,
) -> Result<FileReconstructor, SdxError> {
// Validate the file identifier before spending a token issuance.
let _ = parse_xet_hash_hex(file_id)?;
if let Some(range) = &range
&& range.start > range.end
{
return Err(SdxError::InvalidByteRange {
start: range.start,
end: range.end,
});
}
let token = self.tokens.read_token().await?;
let mut reconstructor =
FileReconstructor::new(self.stream_context()?, file_id.to_owned(), token.token);
if let Some(range) = range {
reconstructor = reconstructor.with_byte_range(range);
}
Ok(reconstructor)
}
/// Returns the number of ranged xorb transfer requests issued so far.
#[must_use]
pub(crate) fn xorb_fetch_count(&self) -> u64 {
self.xorb_fetch_count.load(Ordering::Relaxed)
}
}
/// Sequential download session over one repository.
///
/// Clone is cheap: sessions share the underlying HTTP client and token service.
#[derive(Clone)]
pub struct DownloadSession {
pub(crate) inner: Arc<DownloadSessionInner>,
}
impl DownloadSession {
/// Downloads the file identified by `file_id` (64 lowercase hex
/// characters) to `dest`, returning the number of bytes written.
///
/// Publishes the completed download atomically, replacing only `dest` (not
/// other hard links to its old contents). Unix paths must not contain
/// symlinks. A failure before publication preserves any existing file.
///
/// # Errors
///
/// Returns [`SdxError`] when `file_id` is malformed, token issuance fails,
/// the reconstruction or xorb fetch fails, or the file cannot be written.
pub async fn download_file(&self, file_id: &str, dest: &Path) -> Result<u64, SdxError> {
self.download(file_id, dest, None).await
}
/// Downloads the inclusive byte range `range` of the file identified by
/// `file_id` to `dest`, returning the number of bytes written.
///
/// Range ends are inclusive per the Xet reconstruction contract
/// (`docs/PROTOCOL_CONFORMANCE.md` "Range Semantics").
///
/// # Errors
///
/// Returns [`SdxError`] when `file_id` or the range is invalid, token
/// issuance fails, the reconstruction or xorb fetch fails, the range is
/// past the end of the file, or the file cannot be written.
pub async fn download_range(
&self,
file_id: &str,
range: RangeInclusive<u64>,
dest: &Path,
) -> Result<u64, SdxError> {
let start = *range.start();
let end = *range.end();
if start > end {
return Err(SdxError::InvalidByteRange { start, end });
}
self.download(file_id, dest, Some(ByteRange::new(start, end)))
.await
}
/// Returns a pull-based streaming download of `file_id`.
///
/// `range` is an end-exclusive byte range; `None` means the full file. The
/// returned [`DownloadStream`] yields `Bytes` chunks via
/// [`DownloadStream::next`] / [`DownloadStream::blocking_next`] — the
/// consumer forwards chunks to a socket/stdout and the file is never
/// buffered whole. Memory is bounded by the client's byte-denominated
/// buffer semaphore (see [`crate::XetClientBuilder::with_buffer_semaphore`]).
///
/// The background reconstruction task is spawned paused and auto-starts on
/// the first `next()`/`blocking_next()`; dropping the stream cancels it.
///
/// # Errors
///
/// Returns [`SdxError`] when `file_id` is malformed, token issuance fails,
/// or `range` is inverted (`start > end`). Reconstruction/fetch errors
/// surface on the stream's `next()` calls.
pub async fn download_stream(
&self,
file_id: &str,
range: Option<Range<u64>>,
) -> Result<DownloadStream, SdxError> {
let reconstructor = self.build_reconstructor(file_id, range).await?;
Ok(reconstructor.reconstruct_to_stream())
}
/// Returns a pull-based streaming download of `file_id` that yields
/// `(file_offset, Bytes)` pairs in completion order.
///
/// Progress probes: [`UnorderedDownloadStream::total_bytes_expected`],
/// [`UnorderedDownloadStream::bytes_in_progress`], and
/// [`UnorderedDownloadStream::bytes_completed`].
///
/// # Errors
///
/// Returns [`SdxError`] when `file_id` is malformed, token issuance fails,
/// or `range` is inverted (`start > end`).
pub async fn download_unordered_stream(
&self,
file_id: &str,
range: Option<Range<u64>>,
) -> Result<UnorderedDownloadStream, SdxError> {
let reconstructor = self.build_reconstructor(file_id, range).await?;
Ok(reconstructor.reconstruct_to_unordered_stream())
}
/// Downloads `file_id` into the `std::io::Write` sink `writer` (a file,
/// stdout, ...), running it on a `spawn_blocking` thread and returning the
/// number of bytes written.
///
/// # Errors
///
/// Returns [`SdxError`] when `file_id` is malformed, token issuance fails,
/// a fetch or reconstruction fails, or the writer fails.
pub async fn download_to_writer<W: std::io::Write + Send + 'static>(
&self,
file_id: &str,
writer: W,
) -> Result<u64, SdxError> {
let reconstructor = self.build_reconstructor(file_id, None).await?;
reconstructor.reconstruct_to_writer(writer).await
}
/// Downloads the whole file into a single [`Bytes`] (in-memory convenience
/// path; fine for modest files).
///
/// Equivalent to draining [`download_stream`](Self::download_stream) with
/// `range = None`.
///
/// # Errors
///
/// Returns [`SdxError`] when `file_id` is malformed, token issuance fails,
/// or a fetch/reconstruction fails.
pub async fn download_bytes(&self, file_id: &str) -> Result<Bytes, SdxError> {
let mut stream = self.download_stream(file_id, None).await?;
let mut out = bytes::BytesMut::new();
while let Some(chunk) = stream.next().await? {
out.extend_from_slice(&chunk);
}
Ok(out.freeze())
}
/// Builds a [`FileReconstructor`] for `file_id`, validating the identifier
/// and resolving the read token.
///
/// # Errors
///
/// Returns [`SdxError`] when `file_id` is malformed or token issuance fails.
async fn build_reconstructor(
&self,
file_id: &str,
range: Option<Range<u64>>,
) -> Result<FileReconstructor, SdxError> {
self.inner.build_reconstructor(file_id, range).await
}
async fn download(
&self,
file_id: &str,
dest: &Path,
range: Option<ByteRange>,
) -> Result<u64, SdxError> {
parse_xet_hash_hex(file_id)?;
let token = self.inner.tokens.read_token().await?;
let retry = self.inner.retry_context();
let file = reconstruction::reconstruct(
&self.inner.transfer,
Some(&retry),
&self.inner.api_base,
&token.token,
file_id,
range,
)
.await?;
let len = u64::try_from(file.data.len()).unwrap_or(u64::MAX);
let dest = dest.to_path_buf();
tokio::task::spawn_blocking(move || crate::local_output::write_atomic(&dest, &file.data))
.await
.map_err(|error| SdxError::TaskJoin(error.to_string()))??;
Ok(len)
}
}