use std::{path::PathBuf, sync::Arc};
use url::Url;
use crate::{
auth::{Auth, HttpConfig, RepositoryId},
cache::{ChunkCache, DEFAULT_CHUNK_CACHE_BUDGET_BYTES},
dedup::DedupClient,
error::{SdxError, TransferError},
group::XetStreamGroup,
retry::RetryPolicy,
session::{DownloadSession, DownloadSessionInner},
stream::{
BufferSemaphore, DEFAULT_DOWNLOAD_BUFFER_LIMIT, DEFAULT_DOWNLOAD_BUFFER_SIZE,
DEFAULT_DOWNLOAD_CONCURRENCY, DownloadStream, StreamLimits, UnorderedDownloadStream,
},
transfer::TransferClient,
};
#[derive(Clone)]
pub struct XetClient {
inner: Arc<DownloadSessionInner>,
}
impl XetClient {
pub(crate) const fn download_inner(&self) -> &Arc<DownloadSessionInner> {
&self.inner
}
#[must_use]
pub fn download_session(&self) -> DownloadSession {
DownloadSession {
inner: Arc::clone(&self.inner),
}
}
pub async fn download_stream(
&self,
file_id: &str,
range: Option<std::ops::Range<u64>>,
) -> Result<DownloadStream, SdxError> {
self.download_session()
.download_stream(file_id, range)
.await
}
pub async fn download_unordered_stream(
&self,
file_id: &str,
range: Option<std::ops::Range<u64>>,
) -> Result<UnorderedDownloadStream, SdxError> {
self.download_session()
.download_unordered_stream(file_id, range)
.await
}
pub async fn download_to_writer<W: std::io::Write + Send + 'static>(
&self,
file_id: &str,
writer: W,
) -> Result<u64, SdxError> {
self.download_session()
.download_to_writer(file_id, writer)
.await
}
pub async fn download_bytes(&self, file_id: &str) -> Result<bytes::Bytes, SdxError> {
self.download_session().download_bytes(file_id).await
}
#[must_use]
pub fn cas_url(&self) -> Option<String> {
self.inner.tokens.cas_url()
}
#[must_use]
pub fn new_download_stream_group(&self) -> XetStreamGroup {
XetStreamGroup::new(self.inner.clone())
}
#[must_use]
pub fn xorb_fetch_count(&self) -> u64 {
self.inner.xorb_fetch_count()
}
#[must_use]
pub fn chunk_cache(&self) -> Option<Arc<ChunkCache>> {
self.inner.chunk_cache.clone()
}
#[must_use]
pub fn dedup_client(&self) -> DedupClient {
DedupClient::new(self.inner.transfer.clone())
}
pub fn upload_session(&self) -> Result<crate::upload::UploadSession, SdxError> {
crate::upload::UploadSession::new(&self.inner)
}
pub async fn upload_file(
&self,
path: impl AsRef<std::path::Path>,
remote: &str,
) -> Result<crate::upload::UploadFileInfo, SdxError> {
let session = self.upload_session()?;
let info = session.upload_file(path, remote).await?;
session.finalize().await?;
Ok(info)
}
pub async fn upload_bytes(
&self,
remote: &str,
bytes: impl Into<bytes::Bytes>,
) -> Result<crate::upload::UploadFileInfo, SdxError> {
let session = self.upload_session()?;
let info = session.upload_bytes(remote, bytes).await?;
session.finalize().await?;
Ok(info)
}
pub async fn upload_stream<R: std::io::Read + Send + 'static>(
&self,
remote: &str,
reader: R,
) -> Result<crate::upload::UploadFileInfo, SdxError> {
let session = self.upload_session()?;
let info = session.upload_stream(remote, reader).await?;
session.finalize().await?;
Ok(info)
}
pub fn new_upload_group(&self) -> Result<crate::group::XetUploadCommit, SdxError> {
crate::group::XetUploadCommit::new(&self.inner)
}
}
impl Default for XetClientBuilder {
fn default() -> Self {
Self {
endpoint: None,
auth: None,
http: None,
buffer_capacity: None,
download_concurrency: DEFAULT_DOWNLOAD_CONCURRENCY,
upload_concurrency: crate::upload::DEFAULT_UPLOAD_CONCURRENCY,
limits: StreamLimits::default(),
chunk_cache_dir: None,
chunk_cache: None,
chunk_cache_budget: None,
upload_chunk_size: crate::chunker::DEFAULT_TARGET_CHUNK_SIZE,
retry_policy: RetryPolicy::default(),
session_id: None,
reconstruction_response_limit: 64 * 1024 * 1024,
}
}
}
pub struct XetClientBuilder {
endpoint: Option<String>,
auth: Option<Auth>,
http: Option<HttpConfig>,
buffer_capacity: Option<u64>,
download_concurrency: usize,
upload_concurrency: usize,
limits: StreamLimits,
chunk_cache_dir: Option<PathBuf>,
chunk_cache: Option<Arc<ChunkCache>>,
chunk_cache_budget: Option<u64>,
upload_chunk_size: usize,
retry_policy: RetryPolicy,
session_id: Option<String>,
reconstruction_response_limit: usize,
}
impl XetClientBuilder {
#[must_use]
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub fn endpoint(mut self, endpoint: impl Into<String>) -> Self {
self.endpoint = Some(endpoint.into());
self
}
#[must_use]
pub fn from_url(mut self, url: &crate::url::XetUrl) -> Self {
self.endpoint = Some(url.endpoint_url());
self
}
#[must_use]
pub fn auth(mut self, auth: Auth) -> Self {
self.auth = Some(auth);
self
}
#[must_use]
#[allow(clippy::missing_const_for_fn)]
pub fn http(mut self, http: HttpConfig) -> Self {
self.http = Some(http);
self
}
#[must_use]
pub const fn with_buffer_semaphore(mut self, capacity_bytes: u64) -> Self {
self.buffer_capacity = Some(capacity_bytes);
self
}
#[must_use]
#[allow(clippy::missing_const_for_fn)]
pub fn with_download_concurrency(mut self, concurrency: usize) -> Self {
self.download_concurrency = concurrency.max(1);
self
}
#[must_use]
pub const fn with_stream_limits(mut self, limits: StreamLimits) -> Self {
self.limits = limits;
self
}
#[must_use]
pub const fn with_reconstruction_response_limit(mut self, bytes: usize) -> Self {
self.reconstruction_response_limit = bytes;
self
}
#[must_use]
pub fn with_chunk_cache_dir(mut self, cache_dir: impl Into<PathBuf>) -> Self {
self.chunk_cache_dir = Some(cache_dir.into());
self
}
#[must_use]
pub fn with_chunk_cache(mut self, cache: ChunkCache) -> Self {
self.chunk_cache = Some(Arc::new(cache));
self
}
#[must_use]
pub const fn with_chunk_cache_budget(mut self, budget_bytes: u64) -> Self {
self.chunk_cache_budget = Some(budget_bytes);
self
}
#[must_use]
pub const fn with_upload_chunk_size(mut self, chunk_size: usize) -> Self {
self.upload_chunk_size = chunk_size;
self
}
#[must_use]
#[allow(clippy::missing_const_for_fn)]
pub fn with_upload_concurrency(mut self, concurrency: usize) -> Self {
self.upload_concurrency = concurrency.max(1);
self
}
#[must_use]
pub const fn with_retry_policy(mut self, policy: RetryPolicy) -> Self {
self.retry_policy = policy;
self
}
#[must_use]
pub fn with_session_id(mut self, session_id: impl Into<String>) -> Self {
self.session_id = Some(session_id.into());
self
}
pub fn build(self) -> Result<XetClient, SdxError> {
let endpoint = self.endpoint.ok_or_else(|| {
SdxError::InvalidEndpoint("no endpoint configured; use `.endpoint(...)`".to_owned())
})?;
let (api_base, repository) = parse_endpoint(&endpoint)?;
let auth = self.auth.ok_or_else(|| {
SdxError::InvalidEndpoint("no auth configured; use `.auth(...)`".to_owned())
})?;
let auth_repository = auth.repository();
if &repository != auth_repository {
return Err(SdxError::InvalidEndpoint(format!(
"endpoint repository {repository:?} does not match the auth repository {auth_repository:?}"
)));
}
let tokens = auth.build()?;
let http = self.http.unwrap_or_default();
let http_client = reqwest::Client::builder()
.connect_timeout(http.connect_timeout())
.read_timeout(std::time::Duration::from_secs(300))
.pool_idle_timeout(std::time::Duration::from_secs(60))
.build()
.map_err(TransferError::from)?;
let session_id = self.session_id.clone().unwrap_or_default();
let mut transfer = TransferClient::new(http_client)
.with_reconstruction_response_limit(self.reconstruction_response_limit);
if !session_id.is_empty() {
transfer = transfer.with_session_id(session_id);
}
let (buffer_semaphore, limits) = match self.buffer_capacity {
Some(capacity) => (
Arc::new(BufferSemaphore::new(capacity, capacity, capacity)),
self.limits,
),
None => (
Arc::new(BufferSemaphore::new(
DEFAULT_DOWNLOAD_BUFFER_SIZE,
DEFAULT_DOWNLOAD_BUFFER_SIZE,
DEFAULT_DOWNLOAD_BUFFER_LIMIT,
)),
self.limits,
),
};
let download_permits = Arc::new(tokio::sync::Semaphore::new(
self.download_concurrency.max(1),
));
let chunk_cache = match self.chunk_cache {
Some(cache) => Some(cache),
None => {
let budget = self
.chunk_cache_budget
.unwrap_or(DEFAULT_CHUNK_CACHE_BUDGET_BYTES);
match self.chunk_cache_dir {
Some(dir) => Some(Arc::new(ChunkCache::new(dir, budget)?)),
None => None,
}
}
};
Ok(XetClient {
inner: Arc::new(DownloadSessionInner {
transfer,
tokens,
api_base,
buffer_semaphore,
active_downloads: Arc::new(std::sync::atomic::AtomicU64::new(0)),
download_permits,
limits,
chunk_cache,
xorb_fetch_count: Arc::new(std::sync::atomic::AtomicU64::new(0)),
upload_chunk_size: self.upload_chunk_size,
upload_concurrency: self.upload_concurrency,
retry_policy: self.retry_policy,
repository,
}),
})
}
}
fn parse_endpoint(endpoint: &str) -> Result<(String, RepositoryId), SdxError> {
let url = Url::parse(endpoint).map_err(|error| {
SdxError::InvalidEndpoint(format!("cannot parse {endpoint:?}: {error}"))
})?;
if url.scheme() != "xet" {
return Err(SdxError::InvalidEndpoint(format!(
"unsupported scheme {:?} in {endpoint:?}; expected xet://",
url.scheme()
)));
}
let segments: Vec<&str> = url
.path_segments()
.map(|segments| segments.collect())
.unwrap_or_default();
if segments.len() != 4 {
return Err(SdxError::InvalidEndpoint(format!(
"expected exactly provider/owner/repo/revision in {endpoint:?}, got {} segment(s)",
segments.len()
)));
}
let parsed = crate::url::XetUrl::parse(endpoint)?;
Ok((parsed.api_base.clone(), parsed.repository_id()))
}
#[cfg(test)]
mod tests {
use crate::{Auth, RepositoryId, XetClientBuilder, error::SdxError};
use super::parse_endpoint;
fn repository() -> RepositoryId {
RepositoryId {
provider: "github".to_owned(),
owner: "team".to_owned(),
repo: "assets".to_owned(),
revision: "main".to_owned(),
}
}
#[test]
fn builder_from_url_builds_with_matching_auth_scope() {
let url =
crate::url::XetUrl::parse("xet://example.com/github/team/assets/main/dir/file.txt")
.unwrap();
let auth = Auth::new("http://example.com", repository())
.unwrap()
.with_api_key("bootstrap".to_owned());
let client = XetClientBuilder::new()
.from_url(&url)
.auth(auth)
.build()
.unwrap();
let _ = client;
}
#[test]
fn builder_from_url_mismatched_auth_scope_errors() {
let url = crate::url::XetUrl::parse("xet://example.com/github/other/assets/main").unwrap();
let auth = Auth::new("http://example.com", repository())
.unwrap()
.with_api_key("bootstrap".to_owned());
let error = XetClientBuilder::new()
.from_url(&url)
.auth(auth)
.build()
.err()
.unwrap();
assert!(matches!(error, SdxError::InvalidEndpoint(_)));
}
#[test]
fn parse_endpoint_maps_host_port_and_identity() {
let (base, repository) =
parse_endpoint("xet://127.0.0.1:8080/github/team/assets/main").unwrap();
assert_eq!(base, "http://127.0.0.1:8080");
assert_eq!(
repository,
RepositoryId {
provider: "github".to_owned(),
owner: "team".to_owned(),
repo: "assets".to_owned(),
revision: "main".to_owned(),
}
);
}
#[test]
fn parse_endpoint_defaults_to_port_80() {
let (base, _) = parse_endpoint("xet://example.com/github/team/assets/main").unwrap();
assert_eq!(base, "http://example.com");
}
#[test]
fn parse_endpoint_rejects_wrong_scheme() {
let error = parse_endpoint("http://example.com/github/team/assets/main").unwrap_err();
assert!(matches!(error, SdxError::InvalidEndpoint(_)));
assert!(error.to_string().contains("xet://"));
}
#[test]
fn parse_endpoint_rejects_missing_host() {
let error = parse_endpoint("xet:///github/team/assets/main").unwrap_err();
assert!(matches!(error, SdxError::InvalidEndpoint(_)));
}
#[test]
fn parse_endpoint_rejects_wrong_segment_count() {
let error = parse_endpoint("xet://host/github/team/assets").unwrap_err();
assert!(matches!(error, SdxError::InvalidEndpoint(_)));
let error = parse_endpoint("xet://host/github/team/assets/main/extra").unwrap_err();
assert!(matches!(error, SdxError::InvalidEndpoint(_)));
}
#[test]
fn parse_endpoint_rejects_query_and_fragment() {
let error = parse_endpoint("xet://host/github/team/assets/main?x=1").unwrap_err();
assert!(matches!(error, SdxError::InvalidEndpoint(_)));
let error = parse_endpoint("xet://host/github/team/assets/main#frag").unwrap_err();
assert!(matches!(error, SdxError::InvalidEndpoint(_)));
}
}