use std::io::Read;
use std::path::{Component, Path, PathBuf};
use std::sync::Arc;
use ignore::WalkBuilder;
use reqwest::Method;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use crate::Traced;
use crate::error::SdkError;
use super::ArtifactStorageClient;
#[derive(Clone, Debug)]
pub struct PushFile {
pub repo_path: String,
pub source: PushSource,
pub mode: Option<u32>,
pub delete: bool,
}
#[non_exhaustive]
#[derive(Clone, Debug)]
pub enum PushSource {
Path(PathBuf),
Bytes(Vec<u8>),
KnownOid(String),
StablePrefix {
path: PathBuf,
stable_chunks: Vec<([u8; 32], u32)>,
},
}
#[derive(Clone, Debug)]
pub enum PushEvent {
Chunking {
files_done: usize,
files_total: usize,
bytes_hashed: u64,
},
Hashed {
files: usize,
chunks: usize,
bytes: u64,
},
Negotiated { missing: usize, total: usize },
UploadedBatch { chunks: usize, bytes: u64 },
Committing { files: usize },
CommitDetached { job_id: String },
CommitProgress {
phase: String,
done: u64,
total: u64,
},
Committed { commit: String, ref_name: String },
}
pub type PushProgress = Arc<dyn Fn(PushEvent) + Send + Sync>;
#[derive(Clone)]
pub struct PushOptions {
pub branch: String,
pub message: String,
pub base: Option<String>,
pub expect_oid: Option<String>,
pub upload_batch_bytes: usize,
pub progress: Option<PushProgress>,
pub workspace_snapshot: Option<String>,
pub collect_file_chunks: bool,
pub idempotency_key: Option<String>,
pub on_prepared: Option<PreparedHook>,
}
pub type PreparedHook = Arc<dyn Fn(&[PreparedPush]) + Send + Sync>;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct PreparedPush {
pub path: String,
pub oid: Option<String>,
pub mode: Option<u32>,
pub delete: bool,
}
impl Default for PushOptions {
fn default() -> Self {
PushOptions {
branch: "main".to_string(),
message: String::new(),
base: None,
expect_oid: None,
upload_batch_bytes: 48 * 1024 * 1024,
progress: None,
workspace_snapshot: None,
collect_file_chunks: false,
idempotency_key: None,
on_prepared: None,
}
}
}
#[non_exhaustive]
#[derive(Clone, Debug, Serialize)]
pub struct PushReport {
pub commit: String,
pub tree: String,
pub ref_name: String,
pub created: bool,
pub files: usize,
pub bytes_total: u64,
pub chunks_total: usize,
pub chunks_uploaded: usize,
pub bytes_uploaded: u64,
pub file_blob_oids: Vec<(String, String)>,
#[serde(skip_serializing)]
pub file_chunks: Vec<(String, Vec<([u8; 32], u32)>)>,
}
#[derive(Deserialize)]
struct IngestSessionWire {
session_id: String,
cdc_min_bytes: usize,
cdc_avg_bytes: usize,
cdc_max_bytes: usize,
max_hashes_per_query: usize,
#[allow(dead_code)]
max_chunk_bytes: usize,
#[serde(default)]
features: Vec<String>,
#[serde(default)]
small_file_max_bytes: u64,
#[serde(default)]
max_batch_files: usize,
}
#[derive(Serialize)]
struct MissingWire<'a> {
hashes: &'a [String],
}
#[derive(Deserialize)]
struct MissingRespWire {
missing: Vec<String>,
}
#[derive(Deserialize)]
struct IngestStagingWire {
pack_id: String,
url: Option<String>,
}
#[derive(Deserialize)]
struct BatchUploadRespWire {
files: Vec<BatchFileRespWire>,
}
#[derive(Deserialize)]
struct BatchFileRespWire {
token: String,
oid: String,
#[allow(dead_code)]
#[serde(default)]
deduplicated: bool,
}
#[derive(Serialize)]
struct StagedEntryWire {
hash: String,
offset: u64,
length: u32,
size: u32,
}
#[derive(Serialize)]
struct StagedRegisterWire {
pack_id: String,
entries: Vec<StagedEntryWire>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct CommitJobReadBack {
pub done: u64,
pub total: u64,
}
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
pub struct CommitJobError {
#[serde(default)]
pub kind: String,
#[serde(default)]
pub message: String,
#[serde(default)]
pub retryable: bool,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct CommitJobStatus {
pub job_id: String,
pub state: String,
#[serde(default)]
pub generation: Option<u64>,
#[serde(default)]
pub poll_url: Option<String>,
#[serde(default)]
pub phase: Option<String>,
#[serde(default)]
pub read_back: Option<CommitJobReadBack>,
#[serde(default)]
pub commit: Option<String>,
#[serde(default)]
pub tree: Option<String>,
#[serde(default)]
pub ref_name: Option<String>,
#[serde(default)]
pub parent: Option<String>,
#[serde(default)]
pub created: Option<bool>,
#[serde(default)]
pub error: Option<CommitJobError>,
}
const STAGED_ZSTD_LEVEL: i32 = 3;
const JOB_LONG_POLL_TIMEOUT_MS: u64 = 25_000;
#[derive(Serialize)]
struct CommitFileWire {
path: String,
#[serde(skip_serializing_if = "Vec::is_empty")]
chunks: Vec<CommitChunkWire>,
#[serde(skip_serializing_if = "Option::is_none")]
content: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
file_token: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
oid: Option<String>,
#[serde(skip_serializing_if = "std::ops::Not::not")]
delete: bool,
#[serde(skip_serializing_if = "Option::is_none")]
mode: Option<String>,
}
#[derive(Serialize)]
struct CommitChunkWire {
hash: String,
size: u32,
}
#[derive(Serialize)]
struct CommitWire {
#[serde(skip_serializing_if = "Option::is_none")]
branch: Option<String>,
message: String,
session_id: String,
files: Vec<CommitFileWire>,
#[serde(skip_serializing_if = "Option::is_none")]
base: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
expect_oid: Option<String>,
}
#[derive(Clone, Debug, Deserialize)]
pub struct CommitByReferenceResponse {
pub commit: String,
pub tree: String,
pub ref_name: String,
pub parent: Option<String>,
pub created: bool,
}
struct ChunkedFile {
repo_path: String,
source: PushSource,
chunks: Vec<([u8; 32], u32)>,
mode: Option<u32>,
delete: bool,
known_oid: bool,
blob_oid: String,
cdc: bool,
}
fn chunk_source(
source: &PushSource,
min: usize,
avg: usize,
max: usize,
) -> Result<(Vec<([u8; 32], u32)>, String), SdkError> {
let len: u64 = match source {
PushSource::Path(p) => std::fs::metadata(p).map_err(io_err)?.len(),
PushSource::Bytes(b) => b.len() as u64,
PushSource::KnownOid(_) | PushSource::StablePrefix { .. } => {
return Err(SdkError::ClientError(
"source kind carries no full byte stream to chunk".to_string(),
));
}
};
let reader: Box<dyn Read + Send + '_> = match source {
PushSource::Path(p) => Box::new(std::fs::File::open(p).map_err(io_err)?),
PushSource::Bytes(b) => Box::new(std::io::Cursor::new(b.as_slice())),
PushSource::KnownOid(_) | PushSource::StablePrefix { .. } => {
unreachable!("guarded above")
}
};
let mut blob_hasher = BlobOidHasher::new(len);
let mut out = Vec::new();
for chunk in fastcdc::v2020::StreamCDC::new(reader, min as u32, avg as u32, max as u32) {
let chunk = chunk.map_err(|e| SdkError::ClientError(format!("chunking failed: {e}")))?;
let hash: [u8; 32] = Sha256::digest(&chunk.data).into();
blob_hasher.update(&chunk.data);
out.push((hash, chunk.data.len() as u32));
}
Ok((out, blob_hasher.finalize_hex()))
}
fn chunk_stable_prefix(
path: &Path,
stable_chunks: &[([u8; 32], u32)],
min: usize,
avg: usize,
max: usize,
) -> Result<Vec<([u8; 32], u32)>, SdkError> {
use std::io::Seek as _;
let stable_len: u64 = stable_chunks.iter().map(|(_, s)| *s as u64).sum();
let mut file = std::fs::File::open(path).map_err(io_err)?;
file.seek(std::io::SeekFrom::Start(stable_len))
.map_err(io_err)?;
let mut out = stable_chunks.to_vec();
for chunk in fastcdc::v2020::StreamCDC::new(file, min as u32, avg as u32, max as u32) {
let chunk = chunk.map_err(|e| SdkError::ClientError(format!("chunking failed: {e}")))?;
let hash: [u8; 32] = Sha256::digest(&chunk.data).into();
out.push((hash, chunk.data.len() as u32));
}
Ok(out)
}
fn io_err(e: std::io::Error) -> SdkError {
SdkError::Io(e)
}
pub struct BlobOidHasher(sha1::Sha1);
impl BlobOidHasher {
pub fn new(len: u64) -> BlobOidHasher {
use sha1::Digest as _;
let mut h = sha1::Sha1::new();
h.update(format!("blob {len}\0").as_bytes());
BlobOidHasher(h)
}
pub fn update(&mut self, data: &[u8]) {
use sha1::Digest as _;
self.0.update(data);
}
pub fn finalize_hex(self) -> String {
use sha1::Digest as _;
hex::encode(self.0.finalize())
}
}
const TRANSIENT_RETRIES: usize = 3;
fn is_transient(e: &SdkError) -> bool {
match e {
SdkError::ServerError { status, .. } => status.is_server_error() || status.as_u16() == 429,
SdkError::Http(_) | SdkError::Middleware(_) => true,
_ => false,
}
}
async fn with_transient_retries<T, F, Fut>(mut op: F) -> Result<T, SdkError>
where
F: FnMut() -> Fut,
Fut: std::future::Future<Output = Result<T, SdkError>>,
{
let mut delay = std::time::Duration::from_millis(250);
let mut attempt = 0usize;
loop {
match op().await {
Ok(v) => return Ok(v),
Err(e) if attempt < TRANSIENT_RETRIES && is_transient(&e) => {
attempt += 1;
let jitter = std::time::Duration::from_millis(u64::from(rand::random::<u8>()));
tokio::time::sleep(delay + jitter).await;
delay = (delay * 4).min(std::time::Duration::from_secs(4));
}
Err(e) => return Err(e),
}
}
}
fn chunk_source_whole(source: &PushSource) -> Result<(Vec<([u8; 32], u32)>, String), SdkError> {
let owned: Vec<u8>;
let data: &[u8] = match source {
PushSource::Path(p) => {
owned = std::fs::read(p).map_err(io_err)?;
&owned
}
PushSource::Bytes(b) => b,
PushSource::KnownOid(_) | PushSource::StablePrefix { .. } => {
return Err(SdkError::ClientError(
"source kind carries no full byte stream to hash".to_string(),
));
}
};
let mut blob = BlobOidHasher::new(data.len() as u64);
blob.update(&data);
let chunks = if data.is_empty() {
Vec::new()
} else {
let hash: [u8; 32] = Sha256::digest(&data).into();
vec![(hash, data.len() as u32)]
};
Ok((chunks, blob.finalize_hex()))
}
enum UploadReader<'a> {
File(std::fs::File),
Bytes(std::io::Cursor<&'a [u8]>),
}
impl<'a> UploadReader<'a> {
fn open(source: &'a PushSource) -> Result<UploadReader<'a>, SdkError> {
match source {
PushSource::Path(p) => Ok(UploadReader::File(std::fs::File::open(p).map_err(io_err)?)),
PushSource::StablePrefix { path, .. } => Ok(UploadReader::File(
std::fs::File::open(path).map_err(io_err)?,
)),
PushSource::Bytes(b) => Ok(UploadReader::Bytes(std::io::Cursor::new(b.as_slice()))),
PushSource::KnownOid(_) => Err(SdkError::ClientError(
"known-oid sources carry no bytes to upload".to_string(),
)),
}
}
fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), SdkError> {
match self {
UploadReader::File(f) => f.read_exact(buf).map_err(io_err),
UploadReader::Bytes(c) => c.read_exact(buf).map_err(io_err),
}
}
fn skip(&mut self, n: u64) -> Result<(), SdkError> {
use std::io::Seek as _;
match self {
UploadReader::File(f) => f.seek(std::io::SeekFrom::Current(n as i64)).map(|_| ()),
UploadReader::Bytes(c) => c.seek(std::io::SeekFrom::Current(n as i64)).map(|_| ()),
}
.map_err(io_err)
}
}
fn elect_tokened(
chunked: &[ChunkedFile],
missing: &std::collections::HashSet<[u8; 32]>,
small_max: u64,
) -> Vec<bool> {
chunked
.iter()
.map(|file| {
if file.delete || file.known_oid || file.chunks.is_empty() {
return false;
}
if matches!(file.source, PushSource::StablePrefix { .. }) {
return false;
}
let total: u64 = file.chunks.iter().map(|(_, s)| *s as u64).sum();
if small_max > 0 && total < small_max {
return true;
}
let missing_bytes: u64 = file
.chunks
.iter()
.filter(|(h, _)| missing.contains(h))
.map(|(_, s)| *s as u64)
.sum();
missing_bytes * 2 >= total
})
.collect()
}
fn dedup_needed_chunks(
chunked: &[ChunkedFile],
tokened: &[bool],
missing: &std::collections::HashSet<[u8; 32]>,
) -> std::collections::HashSet<[u8; 32]> {
chunked
.iter()
.zip(tokened)
.filter(|(file, tokened)| !**tokened && !file.delete && !file.known_oid)
.flat_map(|(file, _)| file.chunks.iter().map(|(h, _)| *h))
.filter(|h| missing.contains(h))
.collect()
}
fn collect_worktree_files(root: &Path, out: &mut Vec<PushFile>) -> Result<(), SdkError> {
let mut walker = WalkBuilder::new(root);
walker
.hidden(false)
.ignore(false)
.parents(false)
.require_git(false)
.git_ignore(true)
.git_exclude(true)
.git_global(true)
.follow_links(false)
.sort_by_file_path(|a, b| a.cmp(b))
.filter_entry(|entry| entry.file_name() != ".git");
for entry in walker.build() {
let entry = entry.map_err(|err| SdkError::ClientError(err.to_string()))?;
let path = entry.path();
let meta = std::fs::symlink_metadata(path)?;
if meta.is_dir() {
continue;
}
let repo_path = repo_path_for_worktree(root, path);
if repo_path.is_empty() {
continue;
}
let file_type = meta.file_type();
if file_type.is_symlink() {
let target = std::fs::read_link(path)?;
out.push(PushFile {
repo_path,
source: PushSource::Bytes(symlink_target_bytes(&target)),
mode: Some(0o120000),
delete: false,
});
} else if file_type.is_file() {
out.push(PushFile {
repo_path,
source: PushSource::Path(path.to_path_buf()),
mode: Some(regular_file_mode(&meta)),
delete: false,
});
}
}
Ok(())
}
fn regular_file_mode(meta: &std::fs::Metadata) -> u32 {
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
if meta.permissions().mode() & 0o111 != 0 {
0o100755
} else {
0o100644
}
}
#[cfg(not(unix))]
{
let _ = meta;
0o100644
}
}
fn symlink_target_bytes(target: &Path) -> Vec<u8> {
#[cfg(unix)]
{
use std::os::unix::ffi::OsStrExt;
target.as_os_str().as_bytes().to_vec()
}
#[cfg(not(unix))]
{
target.to_string_lossy().into_owned().into_bytes()
}
}
fn repo_path_for_worktree(root: &Path, path: &Path) -> String {
path.strip_prefix(root)
.unwrap_or(path)
.components()
.filter_map(|c| match c {
Component::Normal(part) => Some(part.to_string_lossy()),
_ => None,
})
.collect::<Vec<_>>()
.join("/")
}
impl ArtifactStorageClient {
#[allow(clippy::too_many_arguments)]
pub async fn push_worktree(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
root: impl AsRef<Path>,
opts: PushOptions,
) -> Result<Traced<PushReport>, SdkError> {
let root = root.as_ref();
let mut files = Vec::new();
collect_worktree_files(root, &mut files)?;
if files.is_empty() {
return Err(SdkError::ClientError(
"no files found under the Git worktree".to_string(),
));
}
self.push_files(project_id, repo, git_username, git_token, files, opts)
.await
}
#[allow(clippy::too_many_arguments)]
pub async fn push_files(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
files: Vec<PushFile>,
opts: PushOptions,
) -> Result<Traced<PushReport>, SdkError> {
let emit = |ev: PushEvent| {
if let Some(p) = &opts.progress {
p(ev)
}
};
let session: IngestSessionWire = with_transient_retries(|| async {
let (req, _trace) = self.git_request(
Method::POST,
project_id,
repo,
Some("ingest/sessions"),
git_username,
git_token,
)?;
expect_json(req.send().await?).await
})
.await?;
let (min, avg, max) = (
session.cdc_min_bytes,
session.cdc_avg_bytes,
session.cdc_max_bytes,
);
let small_max = session.small_file_max_bytes;
let hash_parallelism = std::thread::available_parallelism()
.map(|n| n.get().min(8))
.unwrap_or(4);
let chunked: Vec<ChunkedFile> = {
use futures::StreamExt as _;
let files_total = files.len();
let mut stream = futures::stream::iter(files.into_iter().map(|f| async move {
if f.delete {
return Ok::<ChunkedFile, SdkError>(ChunkedFile {
repo_path: f.repo_path,
source: f.source,
chunks: Vec::new(),
mode: f.mode,
delete: true,
known_oid: false,
blob_oid: String::new(),
cdc: false,
});
}
if let PushSource::KnownOid(oid) = &f.source {
let oid = oid.to_ascii_lowercase();
if oid.len() != 40 || !oid.bytes().all(|b| b.is_ascii_hexdigit()) {
return Err(SdkError::ClientError(format!(
"known oid for {:?} is not a 40-hex git oid: {oid:?}",
f.repo_path
)));
}
return Ok(ChunkedFile {
repo_path: f.repo_path,
blob_oid: oid,
source: f.source,
chunks: Vec::new(),
mode: f.mode,
delete: false,
known_oid: true,
cdc: false,
});
}
let mut f = f;
let source_len: u64 = match &f.source {
PushSource::Path(p) => std::fs::metadata(p).map_err(io_err)?.len(),
PushSource::Bytes(b) => b.len() as u64,
PushSource::StablePrefix { path, .. } => {
std::fs::metadata(path).map_err(io_err)?.len()
}
PushSource::KnownOid(_) => unreachable!("handled above"),
};
let whole = small_max > 0 && source_len < small_max;
if let PushSource::StablePrefix {
path,
stable_chunks,
} = &f.source
{
let stable_len: u64 = stable_chunks.iter().map(|(_, s)| *s as u64).sum();
if whole || stable_chunks.is_empty() || stable_len > source_len {
f.source = PushSource::Path(path.clone());
}
}
tokio::task::spawn_blocking(move || {
let (chunks, blob_oid, cdc) = match &f.source {
PushSource::StablePrefix {
path,
stable_chunks,
} => (
chunk_stable_prefix(path, stable_chunks, min, avg, max)?,
String::new(),
true,
),
source if whole => {
let (chunks, oid) = chunk_source_whole(source)?;
(chunks, oid, false)
}
source => {
let (chunks, oid) = chunk_source(source, min, avg, max)?;
(chunks, oid, true)
}
};
Ok(ChunkedFile {
repo_path: f.repo_path,
source: f.source,
chunks,
mode: f.mode,
delete: false,
known_oid: false,
blob_oid,
cdc,
})
})
.await
.map_err(|e| SdkError::ClientError(format!("chunking task failed: {e}")))?
}))
.buffered(hash_parallelism);
let mut out: Vec<ChunkedFile> = Vec::with_capacity(files_total);
let mut bytes_hashed = 0u64;
let mut last_emit = std::time::Instant::now();
while let Some(result) = stream.next().await {
let file: ChunkedFile = result?;
bytes_hashed += file.chunks.iter().map(|(_, s)| *s as u64).sum::<u64>();
out.push(file);
if last_emit.elapsed() >= std::time::Duration::from_millis(100) {
last_emit = std::time::Instant::now();
emit(PushEvent::Chunking {
files_done: out.len(),
files_total,
bytes_hashed,
});
}
}
out
};
if chunked.iter().any(|f| f.known_oid) && !session.features.iter().any(|f| f == "oid_files")
{
return Err(SdkError::ClientError(
"this artifact-storage server does not support known-oid file references \
(`oid_files` capability missing); re-push the content instead"
.to_string(),
));
}
if let Some(hook) = &opts.on_prepared {
let prepared: Vec<PreparedPush> = chunked
.iter()
.map(|f| PreparedPush {
path: f.repo_path.clone(),
oid: (!f.delete && !f.blob_oid.is_empty()).then(|| f.blob_oid.clone()),
mode: f.mode,
delete: f.delete,
})
.collect();
hook(&prepared);
}
let total_chunks: usize = chunked.iter().map(|f| f.chunks.len()).sum();
let total_bytes: u64 = chunked
.iter()
.flat_map(|f| f.chunks.iter().map(|(_, s)| *s as u64))
.sum();
emit(PushEvent::Hashed {
files: chunked.len(),
chunks: total_chunks,
bytes: total_bytes,
});
let is_small = |f: &ChunkedFile| -> bool {
small_max > 0
&& f.chunks.iter().map(|(_, s)| *s as u64).sum::<u64>() < small_max
&& !f.delete
&& !f.known_oid
&& !f.chunks.is_empty()
};
let mut distinct: Vec<[u8; 32]> = chunked
.iter()
.filter(|f| !is_small(f))
.flat_map(|f| f.chunks.iter().map(|(h, _)| *h))
.collect();
distinct.sort_unstable();
distinct.dedup();
let missing = self
.negotiate_missing(
project_id,
repo,
git_username,
git_token,
&session,
&distinct,
)
.await?;
emit(PushEvent::Negotiated {
missing: missing.len(),
total: distinct.len(),
});
let (mut uploaded_chunks, mut uploaded_bytes) = (0usize, 0u64);
let mut file_tokens: Vec<Option<String>> = vec![None; chunked.len()];
let tokened = elect_tokened(&chunked, &missing, small_max);
{
let mut token_by_oid: std::collections::HashMap<&str, String> =
std::collections::HashMap::new();
let mut owners: Vec<usize> = Vec::new();
for (i, file) in chunked.iter().enumerate() {
if !tokened[i] {
continue;
}
let token = token_by_oid
.entry(file.blob_oid.as_str())
.or_insert_with(|| {
owners.push(i);
format!("f{i}-{}", hex::encode(&file.chunks[0].0[..8]))
});
file_tokens[i] = Some(token.clone());
}
use futures::StreamExt as _;
const TOKENED_UPLOADS_IN_FLIGHT: usize = 4;
let batch_capable = session.features.iter().any(|f| f == "batch_files")
&& session.small_file_max_bytes > 0
&& session.max_batch_files > 0;
let mut batch_owners: Vec<usize> = Vec::new();
let mut single_owners: Vec<usize> = Vec::new();
for i in owners {
let total: u64 = chunked[i].chunks.iter().map(|(_, s)| *s as u64).sum();
if batch_capable && total < session.small_file_max_bytes {
batch_owners.push(i);
} else {
single_owners.push(i);
}
}
const BATCH_GROUP_BYTES: usize = 16 * 1024 * 1024;
let mut groups: Vec<Vec<usize>> = Vec::new();
{
let mut cur: Vec<usize> = Vec::new();
let mut cur_bytes = 0usize;
for &i in &batch_owners {
let file = &chunked[i];
let record_bytes: usize = 14
+ file_tokens[i].as_ref().map(String::len).unwrap_or(0)
+ file
.chunks
.iter()
.map(|(_, s)| 36 + *s as usize)
.sum::<usize>();
if !cur.is_empty()
&& (cur_bytes + record_bytes > BATCH_GROUP_BYTES
|| cur.len() >= session.max_batch_files)
{
groups.push(std::mem::take(&mut cur));
cur_bytes = 0;
}
cur.push(i);
cur_bytes += record_bytes;
}
if !cur.is_empty() {
groups.push(cur);
}
}
let mut batch_results = futures::stream::iter(groups.into_iter().map(|group| {
let session_id = session.session_id.as_str();
let file_tokens = &file_tokens;
let chunked = &chunked;
async move {
let mut body = Vec::new();
let mut group_chunks = 0usize;
let mut group_bytes = 0u64;
let mut expected: Vec<(String, String)> = Vec::new();
for &i in &group {
let file = &chunked[i];
let token = file_tokens[i].clone().expect("owner has a token");
let total: u64 = file.chunks.iter().map(|(_, s)| *s as u64).sum();
body.extend_from_slice(&(token.len() as u16).to_be_bytes());
body.extend_from_slice(&total.to_be_bytes());
body.extend_from_slice(&(file.chunks.len() as u32).to_be_bytes());
body.extend_from_slice(token.as_bytes());
let mut reader: Box<dyn Read + Send + '_> = match &file.source {
PushSource::Path(p) => {
Box::new(std::fs::File::open(p).map_err(io_err)?)
}
PushSource::Bytes(b) => Box::new(std::io::Cursor::new(b.as_slice())),
PushSource::KnownOid(_) | PushSource::StablePrefix { .. } => {
unreachable!("never tokened")
}
};
for (hash, size) in &file.chunks {
let mut data = vec![0u8; *size as usize];
reader.read_exact(&mut data).map_err(io_err)?;
body.extend_from_slice(hash);
body.extend_from_slice(&(data.len() as u32).to_be_bytes());
body.extend_from_slice(&data);
group_chunks += 1;
group_bytes += data.len() as u64;
}
expected.push((token, file.blob_oid.clone()));
}
let resp = self
.put_batch_files(
project_id,
repo,
git_username,
git_token,
session_id,
body,
)
.await?;
let by_token: std::collections::HashMap<&str, &str> = resp
.files
.iter()
.map(|f| (f.token.as_str(), f.oid.as_str()))
.collect();
for (token, oid) in &expected {
match by_token.get(token.as_str()) {
Some(server) if server == oid => {}
Some(server) => {
return Err(SdkError::ClientError(format!(
"server hashed {token:?} to {server} but the client computed {oid}; was the file modified mid-push?"
)));
}
None => {
return Err(SdkError::ClientError(format!(
"batch upload response is missing token {token:?}"
)));
}
}
}
Ok::<(usize, u64), SdkError>((group_chunks, group_bytes))
}
}))
.buffer_unordered(TOKENED_UPLOADS_IN_FLIGHT);
while let Some(done) = batch_results.next().await {
let (chunks, bytes) = done?;
uploaded_chunks += chunks;
uploaded_bytes += bytes;
emit(PushEvent::UploadedBatch {
chunks: uploaded_chunks,
bytes: uploaded_bytes,
});
}
drop(batch_results);
let mut results = futures::stream::iter(single_owners.into_iter().map(|i| {
let file = &chunked[i];
let token = file_tokens[i].clone().expect("owner has a token");
let session_id = session.session_id.as_str();
let batch_bytes = opts.upload_batch_bytes;
async move {
let total: u64 = file.chunks.iter().map(|(_, s)| *s as u64).sum();
let mut reader: Box<dyn Read + Send + '_> = match &file.source {
PushSource::Path(p) => Box::new(std::fs::File::open(p).map_err(io_err)?),
PushSource::Bytes(b) => Box::new(std::io::Cursor::new(b.as_slice())),
PushSource::KnownOid(_) | PushSource::StablePrefix { .. } => {
unreachable!("never tokened")
}
};
let (mut sent_chunks, mut sent_bytes) = (0usize, 0u64);
let mut offset = 0u64;
let mut frame = Vec::with_capacity(batch_bytes + 64);
for (hash, size) in &file.chunks {
let mut data = vec![0u8; *size as usize];
reader.read_exact(&mut data).map_err(io_err)?;
frame.extend_from_slice(hash);
frame.extend_from_slice(&(data.len() as u32).to_be_bytes());
frame.extend_from_slice(&data);
sent_chunks += 1;
sent_bytes += data.len() as u64;
if frame.len() >= batch_bytes {
let sent: u64 = frame_payload_bytes(&frame);
self.put_chunk_frame(
project_id,
repo,
git_username,
git_token,
session_id,
Some((&token, total, offset)),
std::mem::take(&mut frame),
)
.await?;
offset += sent;
}
}
if !frame.is_empty() {
self.put_chunk_frame(
project_id,
repo,
git_username,
git_token,
session_id,
Some((&token, total, offset)),
frame,
)
.await?;
}
Ok::<(usize, u64), SdkError>((sent_chunks, sent_bytes))
}
}))
.buffer_unordered(TOKENED_UPLOADS_IN_FLIGHT);
while let Some(done) = results.next().await {
let (chunks, bytes) = done?;
uploaded_chunks += chunks;
uploaded_bytes += bytes;
emit(PushEvent::UploadedBatch {
chunks: uploaded_chunks,
bytes: uploaded_bytes,
});
}
}
let mut to_upload = dedup_needed_chunks(&chunked, &tokened, &missing);
const DEDUP_UPLOADS_IN_FLIGHT: usize = 3;
let staging_target = if to_upload.is_empty() {
None
} else {
match self
.ingest_staging_target(
project_id,
repo,
git_username,
git_token,
&session.session_id,
)
.await
{
Ok(probe) => probe.url.is_some().then_some(probe),
Err(SdkError::ServerError { status, .. })
if status.as_u16() == 404 || status.as_u16() == 405 =>
{
None
}
Err(e) => return Err(e),
}
};
if let Some(first_target) = staging_target {
use futures::StreamExt as _;
let mut inflight = futures::stream::FuturesUnordered::new();
let mut next_target = Some(first_target);
let mut pack: Vec<u8> = Vec::with_capacity(opts.upload_batch_bytes + 256 * 1024);
let mut entries: Vec<StagedEntryWire> = Vec::new();
for file in &chunked {
if to_upload.is_empty() {
break;
}
if file.chunks.is_empty() {
continue; }
let mut reader = UploadReader::open(&file.source)?;
for (hash, size) in &file.chunks {
if !to_upload.remove(hash) {
reader.skip(*size as u64)?;
continue;
}
let mut data = vec![0u8; *size as usize];
reader.read_exact(&mut data)?;
let zframe = zstd::encode_all(&data[..], STAGED_ZSTD_LEVEL)
.map_err(|e| SdkError::ClientError(format!("zstd encode: {e}")))?;
entries.push(StagedEntryWire {
hash: hex_lower(hash),
offset: pack.len() as u64,
length: zframe.len() as u32,
size: *size,
});
pack.extend_from_slice(&zframe);
uploaded_chunks += 1;
uploaded_bytes += data.len() as u64;
if pack.len() >= opts.upload_batch_bytes {
let target = match next_target.take() {
Some(t) => t,
None => {
self.ingest_staging_target(
project_id,
repo,
git_username,
git_token,
&session.session_id,
)
.await?
}
};
inflight.push(self.upload_staged_pack(
project_id,
repo,
git_username,
git_token,
&session.session_id,
target,
std::mem::take(&mut pack),
std::mem::take(&mut entries),
));
emit(PushEvent::UploadedBatch {
chunks: uploaded_chunks,
bytes: uploaded_bytes,
});
if inflight.len() >= DEDUP_UPLOADS_IN_FLIGHT {
inflight.next().await.expect("inflight upload present")?;
}
}
}
}
if !pack.is_empty() {
let target = match next_target.take() {
Some(t) => t,
None => {
self.ingest_staging_target(
project_id,
repo,
git_username,
git_token,
&session.session_id,
)
.await?
}
};
inflight.push(self.upload_staged_pack(
project_id,
repo,
git_username,
git_token,
&session.session_id,
target,
pack,
entries,
));
emit(PushEvent::UploadedBatch {
chunks: uploaded_chunks,
bytes: uploaded_bytes,
});
}
while let Some(done) = inflight.next().await {
done?;
}
} else {
use futures::StreamExt as _;
let mut inflight = futures::stream::FuturesUnordered::new();
let mut frame = Vec::with_capacity(opts.upload_batch_bytes + 64);
for file in &chunked {
if to_upload.is_empty() {
break;
}
if file.chunks.is_empty() {
continue; }
let mut reader = UploadReader::open(&file.source)?;
for (hash, size) in &file.chunks {
if !to_upload.remove(hash) {
reader.skip(*size as u64)?;
continue;
}
let mut data = vec![0u8; *size as usize];
reader.read_exact(&mut data)?;
frame.extend_from_slice(hash);
frame.extend_from_slice(&(data.len() as u32).to_be_bytes());
frame.extend_from_slice(&data);
uploaded_chunks += 1;
uploaded_bytes += data.len() as u64;
if frame.len() >= opts.upload_batch_bytes {
inflight.push(self.put_chunk_frame(
project_id,
repo,
git_username,
git_token,
&session.session_id,
None,
std::mem::take(&mut frame),
));
emit(PushEvent::UploadedBatch {
chunks: uploaded_chunks,
bytes: uploaded_bytes,
});
if inflight.len() >= DEDUP_UPLOADS_IN_FLIGHT {
inflight.next().await.expect("inflight upload present")?;
}
}
}
}
if !frame.is_empty() {
inflight.push(self.put_chunk_frame(
project_id,
repo,
git_username,
git_token,
&session.session_id,
None,
frame,
));
emit(PushEvent::UploadedBatch {
chunks: uploaded_chunks,
bytes: uploaded_bytes,
});
}
while let Some(done) = inflight.next().await {
done?;
}
}
let token_commits = session.features.iter().any(|f| f == "token_commits");
let commit_files: Vec<CommitFileWire> = chunked
.iter()
.enumerate()
.map(|(i, f)| {
let token_only = token_commits && file_tokens[i].is_some() && is_small(f);
CommitFileWire {
path: f.repo_path.clone(),
chunks: if token_only {
Vec::new()
} else {
f.chunks
.iter()
.map(|(h, s)| CommitChunkWire {
hash: hex_lower(h),
size: *s,
})
.collect()
},
content: (!f.delete && !f.known_oid && !token_only && f.chunks.is_empty())
.then(String::new),
file_token: file_tokens[i].clone(),
oid: (f.known_oid || token_only).then(|| f.blob_oid.clone()),
delete: f.delete,
mode: f.mode.map(|m| format!("{m:o}")),
}
})
.collect();
let (commit_suffix, branch) = match &opts.workspace_snapshot {
Some(ws_id) => (
format!("workspaces/{}/snapshots", super::encode_path_segment(ws_id)),
None,
),
None => ("commits".to_string(), Some(opts.branch.clone())),
};
let body = CommitWire {
branch,
message: opts.message.clone(),
session_id: session.session_id.clone(),
files: commit_files,
base: opts.base.clone(),
expect_oid: opts.expect_oid.clone(),
};
emit(PushEvent::Committing {
files: body.files.len(),
});
let idem_key = opts.idempotency_key.clone().unwrap_or_else(|| {
let bytes: [u8; 16] = rand::random();
bytes.iter().map(|b| format!("{b:02x}")).collect::<String>()
});
let (accepted, mut job, trace_id) = with_transient_retries(|| async {
let (req, trace_id) = self.git_request(
Method::POST,
project_id,
repo,
Some(&commit_suffix),
git_username,
git_token,
)?;
let submit = req
.header("x-commit-async", "1")
.header("idempotency-key", &idem_key)
.timeout(std::time::Duration::from_secs(60))
.json(&body)
.send()
.await?;
let accepted = submit.status().as_u16() == 202;
let job: CommitJobStatus = expect_json(submit).await?;
Ok((accepted, job, trace_id))
})
.await?;
if accepted {
let job_id = job.job_id.clone();
emit(PushEvent::CommitDetached {
job_id: job_id.clone(),
});
let mut delay = std::time::Duration::from_millis(500);
let mut last_gen: Option<u64> = job.generation;
loop {
job = with_transient_retries(|| async {
self.commit_job_get(
project_id,
repo,
git_username,
git_token,
&job_id,
last_gen,
)
.await
.map(Traced::into_inner)
})
.await?;
match job.state.as_str() {
"committed" | "failed" => break,
_ => {
let (done, total) = job
.read_back
.as_ref()
.map_or((0, 0), |rb| (rb.done, rb.total));
emit(PushEvent::CommitProgress {
phase: job.phase.clone().unwrap_or_else(|| job.state.clone()),
done,
total,
});
}
}
let advanced = match (last_gen, job.generation) {
(Some(prev), Some(cur)) => cur > prev,
(None, Some(_)) => true, _ => false,
};
if job.generation.is_some() {
last_gen = job.generation;
}
if advanced {
delay = std::time::Duration::from_millis(500);
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
} else {
tokio::time::sleep(delay).await;
delay = (delay * 2).min(std::time::Duration::from_secs(2));
}
}
}
if job.state == "failed" {
let err = job.error.unwrap_or_default();
return Err(SdkError::ClientError(format!(
"commit job failed ({}): {}{}",
err.kind,
err.message,
if err.retryable {
" (safe to retry: uploaded chunks are deduplicated)"
} else {
""
}
)));
}
let resp = CommitByReferenceResponse {
commit: job.commit.ok_or_else(|| {
SdkError::ClientError("committed job missing commit oid".to_string())
})?,
tree: job
.tree
.ok_or_else(|| SdkError::ClientError("committed job missing tree".to_string()))?,
ref_name: job.ref_name.ok_or_else(|| {
SdkError::ClientError("committed job missing ref name".to_string())
})?,
parent: job.parent,
created: job.created.unwrap_or(false),
};
emit(PushEvent::Committed {
commit: resp.commit.clone(),
ref_name: resp.ref_name.clone(),
});
Ok(Traced::new(
trace_id,
PushReport {
commit: resp.commit,
tree: resp.tree,
ref_name: resp.ref_name,
created: resp.created,
files: chunked.len(),
bytes_total: total_bytes,
chunks_total: distinct.len(),
chunks_uploaded: uploaded_chunks,
bytes_uploaded: uploaded_bytes,
file_chunks: if opts.collect_file_chunks {
chunked
.iter()
.filter(|f| !f.delete && !f.known_oid && f.cdc)
.map(|f| (f.repo_path.clone(), f.chunks.clone()))
.collect()
} else {
Vec::new()
},
file_blob_oids: chunked
.into_iter()
.map(|f| (f.repo_path, f.blob_oid))
.collect(),
},
))
}
async fn negotiate_missing(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
session: &IngestSessionWire,
distinct: &[[u8; 32]],
) -> Result<std::collections::HashSet<[u8; 32]>, SdkError> {
let mut missing = std::collections::HashSet::new();
for batch in distinct.chunks(session.max_hashes_per_query.max(1)) {
let hashes: Vec<String> = batch.iter().map(hex_lower).collect();
let resp: MissingRespWire = with_transient_retries(|| async {
let (req, _t) = self.git_request(
Method::POST,
project_id,
repo,
Some(&format!("ingest/sessions/{}/missing", session.session_id)),
git_username,
git_token,
)?;
expect_json(req.json(&MissingWire { hashes: &hashes }).send().await?).await
})
.await?;
for h in resp.missing {
let mut arr = [0u8; 32];
hex::decode_to_slice(&h, &mut arr)
.map_err(|e| SdkError::ClientError(format!("bad hash from server: {e}")))?;
missing.insert(arr);
}
}
Ok(missing)
}
pub async fn commit_job_status(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
job_id: &str,
) -> Result<Traced<CommitJobStatus>, SdkError> {
self.commit_job_get(project_id, repo, git_username, git_token, job_id, None)
.await
}
async fn commit_job_get(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
job_id: &str,
wait_generation: Option<u64>,
) -> Result<Traced<CommitJobStatus>, SdkError> {
let id = super::encode_path_segment(job_id);
let (suffix, timeout) = match wait_generation {
Some(g) => (
format!(
"commits/jobs/{id}?wait_generation={g}&timeout_ms={JOB_LONG_POLL_TIMEOUT_MS}"
),
std::time::Duration::from_millis(JOB_LONG_POLL_TIMEOUT_MS + 10_000),
),
None => (
format!("commits/jobs/{id}"),
std::time::Duration::from_secs(30),
),
};
let (req, trace_id) = self.git_request(
Method::GET,
project_id,
repo,
Some(&suffix),
git_username,
git_token,
)?;
let value = expect_json(req.timeout(timeout).send().await?).await?;
Ok(Traced::new(trace_id, value))
}
async fn ingest_staging_target(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
session_id: &str,
) -> Result<IngestStagingWire, SdkError> {
with_transient_retries(|| async {
let (req, _t) = self.git_request(
Method::POST,
project_id,
repo,
Some(&format!("ingest/sessions/{session_id}/staging")),
git_username,
git_token,
)?;
expect_json(req.send().await?).await
})
.await
}
#[allow(clippy::too_many_arguments)]
async fn upload_staged_pack(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
session_id: &str,
target: IngestStagingWire,
pack: Vec<u8>,
entries: Vec<StagedEntryWire>,
) -> Result<(), SdkError> {
let url = target.url.as_deref().ok_or_else(|| {
SdkError::ClientError("staged upload requires a presigned url".to_string())
})?;
let pack = bytes::Bytes::from(pack);
with_transient_retries(|| {
let pack = pack.clone();
async move {
let resp = self.git_client.put(url).body(pack).send().await?;
if !resp.status().is_success() {
return Err(SdkError::ServerError {
status: resp.status(),
message: format!(
"staged pack PUT: {}",
resp.text().await.unwrap_or_default()
),
});
}
Ok(())
}
})
.await?;
const REGISTER_BATCH: usize = 4096;
let mut remaining = entries;
while !remaining.is_empty() {
let take = remaining.len().min(REGISTER_BATCH);
let batch: Vec<StagedEntryWire> = remaining.drain(..take).collect();
let body = StagedRegisterWire {
pack_id: target.pack_id.clone(),
entries: batch,
};
with_transient_retries(|| async {
let (req, _t) = self.git_request(
Method::POST,
project_id,
repo,
Some(&format!("ingest/sessions/{session_id}/staged")),
git_username,
git_token,
)?;
expect_ok(req.json(&body).send().await?).await
})
.await?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
async fn put_batch_files(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
session_id: &str,
body: Vec<u8>,
) -> Result<BatchUploadRespWire, SdkError> {
let body = bytes::Bytes::from(body);
with_transient_retries(|| {
let body = body.clone();
async move {
let (req, _t) = self.git_request(
Method::PUT,
project_id,
repo,
Some(&format!("ingest/sessions/{session_id}/files")),
git_username,
git_token,
)?;
expect_json(req.body(body).send().await?).await
}
})
.await
}
#[allow(clippy::too_many_arguments)]
async fn put_chunk_frame(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
session_id: &str,
file: Option<(&str, u64, u64)>,
frame: Vec<u8>,
) -> Result<(), SdkError> {
let suffix = match file {
None => format!("ingest/sessions/{session_id}/chunks"),
Some((token, total, offset)) => format!(
"ingest/sessions/{session_id}/chunks?file={token}&file_total={total}&file_offset={offset}"
),
};
let retriable = file.is_none();
let frame = bytes::Bytes::from(frame);
let send = || {
let frame = frame.clone();
let suffix = suffix.clone();
async move {
let (req, _t) = self.git_request(
Method::PUT,
project_id,
repo,
Some(&suffix),
git_username,
git_token,
)?;
expect_ok(req.body(frame).send().await?).await
}
};
if retriable {
with_transient_retries(send).await
} else {
send().await
}
}
}
fn frame_payload_bytes(frame: &[u8]) -> u64 {
let mut cursor = frame;
let mut total = 0u64;
while cursor.len() >= 36 {
let len = u32::from_be_bytes(cursor[32..36].try_into().expect("sized")) as u64;
total += len;
cursor = &cursor[36 + len as usize..];
}
total
}
fn hex_lower(h: &[u8; 32]) -> String {
hex::encode(h)
}
pub(super) async fn expect_json<T: serde::de::DeserializeOwned>(
resp: reqwest::Response,
) -> Result<T, SdkError> {
let status = resp.status();
if !status.is_success() {
let message = resp.text().await.unwrap_or_default();
return Err(SdkError::ServerError { status, message });
}
resp.json::<T>()
.await
.map_err(|e| SdkError::ClientError(format!("bad response body: {e}")))
}
async fn expect_ok(resp: reqwest::Response) -> Result<(), SdkError> {
let status = resp.status();
if !status.is_success() {
let message = resp.text().await.unwrap_or_default();
return Err(SdkError::ServerError { status, message });
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn cf(chunks: Vec<([u8; 32], u32)>, delete: bool, known_oid: bool) -> ChunkedFile {
let cdc = !delete && !known_oid && !chunks.is_empty();
ChunkedFile {
repo_path: "p".to_string(),
source: PushSource::Bytes(Vec::new()),
chunks,
mode: None,
delete,
known_oid,
blob_oid: String::new(),
cdc,
}
}
fn h(b: u8) -> [u8; 32] {
[b; 32]
}
#[tokio::test(start_paused = true)]
async fn transient_retries_back_off_and_stop_on_client_errors() {
use std::sync::atomic::{AtomicUsize, Ordering};
let calls = AtomicUsize::new(0);
let out: Result<u32, SdkError> = with_transient_retries(|| {
let n = calls.fetch_add(1, Ordering::SeqCst);
async move {
if n < 2 {
Err(SdkError::ServerError {
status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
message: "wobble".into(),
})
} else {
Ok(7)
}
}
})
.await;
assert_eq!(out.unwrap(), 7);
assert_eq!(
calls.load(Ordering::SeqCst),
3,
"two transient failures retried"
);
let calls = AtomicUsize::new(0);
let out: Result<u32, SdkError> = with_transient_retries(|| {
calls.fetch_add(1, Ordering::SeqCst);
async {
Err(SdkError::ServerError {
status: reqwest::StatusCode::BAD_REQUEST,
message: "no".into(),
})
}
})
.await;
assert!(out.is_err());
assert_eq!(
calls.load(Ordering::SeqCst),
1,
"4xx must not retry: the rejection is deterministic"
);
let calls = AtomicUsize::new(0);
let out: Result<u32, SdkError> = with_transient_retries(|| {
calls.fetch_add(1, Ordering::SeqCst);
async {
Err(SdkError::ServerError {
status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
message: "down".into(),
})
}
})
.await;
assert!(out.is_err());
assert_eq!(calls.load(Ordering::SeqCst), 1 + TRANSIENT_RETRIES);
}
#[test]
fn tokened_election_smalls_always_larges_by_ratio() {
let missing: std::collections::HashSet<[u8; 32]> = [h(1), h(2)].into();
let files = vec![
cf(vec![(h(9), 100)], false, false), cf(vec![(h(1), 300), (h(9), 300)], false, false), cf(vec![(h(9), 300), (h(8), 300)], false, false), cf(vec![(h(2), 100)], true, false), cf(Vec::new(), false, true), cf(Vec::new(), false, false), ];
assert_eq!(
elect_tokened(&files, &missing, 200),
vec![true, true, false, false, false, false]
);
assert_eq!(
elect_tokened(&files, &missing, 0),
vec![false, true, false, false, false, false]
);
}
#[test]
fn dedup_coverage_ignores_tokened_uploads() {
let missing: std::collections::HashSet<[u8; 32]> = [h(1), h(2), h(3)].into();
let files = vec![
cf(vec![(h(1), 100), (h(2), 100)], false, false),
cf(vec![(h(1), 100), (h(3), 100), (h(9), 800)], false, false),
];
let tokened = vec![true, false];
let need = dedup_needed_chunks(&files, &tokened, &missing);
assert!(need.contains(&h(1)), "shared chunk must upload via dedup");
assert!(need.contains(&h(3)));
assert!(
!need.contains(&h(2)),
"chunks only a tokened file carries ride the tokened stream"
);
}
#[test]
fn worktree_repo_paths_from_dot_have_no_dot_components() {
let root = std::path::Path::new(".");
assert_eq!(
repo_path_for_worktree(root, std::path::Path::new("./.clang-format")),
".clang-format"
);
assert_eq!(
repo_path_for_worktree(root, std::path::Path::new("./arch/boot.c")),
"arch/boot.c"
);
}
#[test]
fn worktree_repo_paths_from_absolute_root_are_root_relative() {
let root = std::path::Path::new("/tmp/somedir");
assert_eq!(
repo_path_for_worktree(root, std::path::Path::new("/tmp/somedir/a/b.txt")),
"a/b.txt"
);
}
#[test]
fn collect_worktree_files_skips_git_metadata() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("a.txt"), "a").unwrap();
std::fs::create_dir(dir.path().join(".git")).unwrap();
std::fs::write(dir.path().join(".git").join("config"), "ignored").unwrap();
let mut files = Vec::new();
collect_worktree_files(dir.path(), &mut files).unwrap();
assert_eq!(files.len(), 1);
assert_eq!(files[0].repo_path, "a.txt");
}
#[test]
fn collect_worktree_files_honors_gitignore_without_git_dir() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join(".gitignore"), "ignored.bin\n").unwrap();
std::fs::write(dir.path().join("ignored.bin"), "ignored").unwrap();
std::fs::write(dir.path().join("kept.tensorlake-test"), "kept").unwrap();
let mut files = Vec::new();
collect_worktree_files(dir.path(), &mut files).unwrap();
let paths: Vec<_> = files.iter().map(|file| file.repo_path.as_str()).collect();
assert_eq!(paths, vec![".gitignore", "kept.tensorlake-test"]);
}
#[cfg(unix)]
#[test]
fn collect_worktree_files_preserves_executable_mode() {
use std::os::unix::fs::PermissionsExt;
let dir = tempfile::tempdir().unwrap();
let script = dir.path().join("script.sh");
std::fs::write(&script, "#!/bin/sh\n").unwrap();
std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap();
let mut files = Vec::new();
collect_worktree_files(dir.path(), &mut files).unwrap();
assert_eq!(files.len(), 1);
assert_eq!(files[0].repo_path, "script.sh");
assert_eq!(files[0].mode, Some(0o100755));
}
#[cfg(unix)]
#[test]
fn collect_worktree_files_preserves_symlink_mode_and_target() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("target.txt"), "target").unwrap();
std::os::unix::fs::symlink("target.txt", dir.path().join("link.txt")).unwrap();
let mut files = Vec::new();
collect_worktree_files(dir.path(), &mut files).unwrap();
let link = files
.iter()
.find(|file| file.repo_path == "link.txt")
.expect("symlink should be collected");
assert_eq!(link.mode, Some(0o120000));
match &link.source {
PushSource::Bytes(bytes) => assert_eq!(bytes, b"target.txt"),
other => panic!("expected symlink bytes, got {other:?}"),
}
}
#[test]
fn known_oid_commit_wire_shape() {
let wire = CommitFileWire {
path: "copies/b.txt".to_string(),
chunks: Vec::new(),
content: None,
file_token: None,
oid: Some("00112233445566778899aabbccddeeff00112233".to_string()),
delete: false,
mode: None,
};
let json = serde_json::to_value(&wire).unwrap();
assert_eq!(json["oid"], "00112233445566778899aabbccddeeff00112233");
assert!(json.get("chunks").is_none());
assert!(json.get("content").is_none());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn detached_snapshot_push_polls_commit_job_route() {
use std::sync::{Arc, Mutex};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
fn respond(path: &str) -> (&'static str, String) {
if path.ends_with("/ingest/sessions") {
(
"200 OK",
serde_json::json!({
"session_id": "s1",
"cdc_min_bytes": 1024,
"cdc_avg_bytes": 4096,
"cdc_max_bytes": 16384,
"max_hashes_per_query": 512,
"max_chunk_bytes": 1_048_576,
"features": ["oid_files"],
})
.to_string(),
)
} else if path.contains("/workspaces/ws-1/snapshots") {
(
"202 Accepted",
serde_json::json!({
"job_id": "j-1",
"state": "running",
"generation": 1,
"poll_url": "/project/p/repos/r/commits/jobs/j-1",
})
.to_string(),
)
} else if path.contains("/commits/jobs/j-1") {
(
"200 OK",
serde_json::json!({
"job_id": "j-1",
"state": "committed",
"generation": 3,
"commit": "c".repeat(40),
"tree": "d".repeat(40),
"ref_name": "refs/workspaces/ws-1",
"created": true,
})
.to_string(),
)
} else if path.ends_with("/commits") {
(
"201 Created",
serde_json::json!({
"job_id": "j-2",
"state": "committed",
"generation": 2,
"commit": "e".repeat(40),
"tree": "d".repeat(40),
"ref_name": "refs/heads/main",
"created": true,
})
.to_string(),
)
} else {
(
"404 Not Found",
serde_json::json!({ "message": "no such route (test server)" }).to_string(),
)
}
}
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let seen: Arc<Mutex<Vec<String>>> = Arc::default();
let seen_srv = seen.clone();
let server = tokio::spawn(async move {
loop {
let Ok((mut sock, _)) = listener.accept().await else {
return;
};
let seen = seen_srv.clone();
tokio::spawn(async move {
let mut buf: Vec<u8> = Vec::new();
loop {
let head_end = loop {
if let Some(pos) = buf.windows(4).position(|w| w == b"\r\n\r\n") {
break pos + 4;
}
let mut tmp = [0u8; 4096];
match sock.read(&mut tmp).await {
Ok(0) | Err(_) => return,
Ok(n) => buf.extend_from_slice(&tmp[..n]),
}
};
let head = String::from_utf8_lossy(&buf[..head_end]).to_string();
let request_line = head.lines().next().unwrap_or_default().to_string();
let content_length: usize = head
.lines()
.find_map(|l| {
l.to_ascii_lowercase()
.strip_prefix("content-length:")
.map(|v| v.trim().parse().unwrap_or(0))
})
.unwrap_or(0);
while buf.len() < head_end + content_length {
let mut tmp = [0u8; 4096];
match sock.read(&mut tmp).await {
Ok(0) | Err(_) => return,
Ok(n) => buf.extend_from_slice(&tmp[..n]),
}
}
buf.drain(..head_end + content_length);
seen.lock().unwrap().push(request_line.clone());
let path = request_line
.split_whitespace()
.nth(1)
.unwrap_or_default()
.to_string();
let (status, body) = respond(&path);
let resp = format!(
"HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\n\r\n{body}",
body.len()
);
if sock.write_all(resp.as_bytes()).await.is_err() {
return;
}
}
});
}
});
let base = format!("http://{addr}");
let client = crate::ClientBuilder::new(&base)
.bearer_token("dummy")
.build()
.unwrap();
let sdk = ArtifactStorageClient::new(client, &base).unwrap();
let report = sdk
.push_files(
"p",
"r",
"u",
"tok",
vec![PushFile {
repo_path: "a.txt".to_string(),
source: PushSource::KnownOid("a".repeat(40)),
mode: None,
delete: false,
}],
PushOptions {
message: "m".to_string(),
workspace_snapshot: Some("ws-1".to_string()),
..Default::default()
},
)
.await
.expect("push must survive a 202 detach")
.into_inner();
let branch_report = sdk
.push_files(
"p",
"r",
"u",
"tok",
vec![PushFile {
repo_path: "b.txt".to_string(),
source: PushSource::KnownOid("b".repeat(40)),
mode: None,
delete: false,
}],
PushOptions {
message: "m2".to_string(),
branch: "main".to_string(),
..Default::default()
},
)
.await
.expect("branch push must succeed")
.into_inner();
server.abort();
assert_eq!(report.commit, "c".repeat(40));
assert_eq!(branch_report.commit, "e".repeat(40));
let seen = seen.lock().unwrap().clone();
assert!(
seen.iter()
.any(|l| l.starts_with("POST /project/p/repos/r/workspaces/ws-1/snapshots")),
"snapshot submit must use the workspace route; saw: {seen:?}"
);
assert!(
seen.iter()
.any(|l| l.starts_with("POST /project/p/repos/r/commits HTTP")),
"branch submit must use the commits route; saw: {seen:?}"
);
let polls: Vec<&String> = seen.iter().filter(|l| l.starts_with("GET ")).collect();
assert!(
!polls.is_empty(),
"a 202 detach must be followed by at least one poll; saw: {seen:?}"
);
for poll in polls {
assert!(
poll.starts_with("GET /project/p/repos/r/commits/jobs/j-1"),
"detached job polled the wrong route: {poll}"
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "requires a local artifact-storage server on 127.0.0.1:8080"]
async fn push_paths_roundtrip_against_local_server() {
const BASE: &str = "http://127.0.0.1:8080";
if std::net::TcpStream::connect_timeout(
&"127.0.0.1:8080".parse().unwrap(),
std::time::Duration::from_millis(500),
)
.is_err()
{
eprintln!("skipping: no local artifact-storage server");
return;
}
let client = crate::ClientBuilder::new(BASE)
.bearer_token("dummy")
.build()
.unwrap();
let sdk = ArtifactStorageClient::new(client, BASE).unwrap();
let repo = format!(
"ingest-{}",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
);
sdk.create_repo_with_credential("ingesttest", &repo, None, None, "t", "devtoken")
.await
.unwrap();
let salt = repo.as_bytes().to_vec();
let content_a: Vec<u8> = [b"alpha ".as_slice(), &salt].concat();
let big: Vec<u8> = (0..3usize * 1024 * 1024)
.map(|i| {
(i as u8)
.wrapping_mul(31)
.wrapping_add(salt[i % salt.len()])
})
.collect();
let report = sdk
.push_files(
"ingesttest",
&repo,
"t",
"devtoken",
vec![
PushFile {
repo_path: "a.txt".to_string(),
source: PushSource::Bytes(content_a.clone()),
mode: None,
delete: false,
},
PushFile {
repo_path: "dup/a-again.txt".to_string(),
source: PushSource::Bytes(content_a.clone()),
mode: None,
delete: false,
},
PushFile {
repo_path: "big.bin".to_string(),
source: PushSource::Bytes(big),
mode: None,
delete: false,
},
PushFile {
repo_path: "empty.txt".to_string(),
source: PushSource::Bytes(Vec::new()),
mode: None,
delete: false,
},
],
PushOptions {
message: "seed".into(),
..Default::default()
},
)
.await
.unwrap()
.into_inner();
assert_eq!(report.files, 4);
assert_eq!(
report
.file_blob_oids
.iter()
.find(|(p, _)| p == "empty.txt")
.unwrap()
.1,
"e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
"empty file must carry git's empty-blob oid"
);
let a_oid = report
.file_blob_oids
.iter()
.find(|(p, _)| p == "a.txt")
.unwrap()
.1
.clone();
let dup_oid = &report
.file_blob_oids
.iter()
.find(|(p, _)| p == "dup/a-again.txt")
.unwrap()
.1;
assert_eq!(&a_oid, dup_oid, "identical content has one blob oid");
let report2 = sdk
.push_files(
"ingesttest",
&repo,
"t",
"devtoken",
vec![PushFile {
repo_path: "copies/by-ref.txt".to_string(),
source: PushSource::KnownOid(a_oid.clone()),
mode: None,
delete: false,
}],
PushOptions {
message: "by reference".into(),
..Default::default()
},
)
.await
.unwrap()
.into_inner();
assert_eq!(report2.bytes_uploaded, 0, "oid references upload nothing");
assert_eq!(
report2.file_blob_oids,
vec![("copies/by-ref.txt".to_string(), a_oid)]
);
let bogus = sdk
.push_files(
"ingesttest",
&repo,
"t",
"devtoken",
vec![PushFile {
repo_path: "copies/bogus.txt".to_string(),
source: PushSource::KnownOid(
"00112233445566778899aabbccddeeff00112233".to_string(),
),
mode: None,
delete: false,
}],
PushOptions {
message: "bogus".into(),
..Default::default()
},
)
.await;
assert!(bogus.is_err(), "unknown oid must fail the commit");
}
#[test]
fn chunking_is_deterministic_and_frames_are_well_formed() {
let data: Vec<u8> = (0..3_000_000usize).map(|i| (i % 251) as u8).collect();
let src = PushSource::Bytes(data.clone());
let (a, oid_a) = chunk_source(&src, 256 * 1024, 1024 * 1024, 4 * 1024 * 1024).unwrap();
let (b, oid_b) = chunk_source(&src, 256 * 1024, 1024 * 1024, 4 * 1024 * 1024).unwrap();
assert_eq!(a, b, "CDC must be deterministic across passes");
assert_eq!(oid_a, oid_b, "blob oid must be deterministic");
let mut reference = BlobOidHasher::new(data.len() as u64);
reference.update(&data);
assert_eq!(oid_a, reference.finalize_hex());
assert_eq!(
a.iter().map(|(_, s)| *s as usize).sum::<usize>(),
data.len(),
"chunks must cover the file exactly"
);
let (hash, size) = a[0];
let mut frame = Vec::new();
frame.extend_from_slice(&hash);
frame.extend_from_slice(&size.to_be_bytes());
frame.extend_from_slice(&data[..size as usize]);
assert_eq!(&frame[..32], &hash);
assert_eq!(u32::from_be_bytes(frame[32..36].try_into().unwrap()), size);
assert_eq!(
<[u8; 32]>::from(Sha256::digest(&frame[36..])),
hash,
"framed bytes must hash to the declared id"
);
}
#[test]
fn whole_file_pass_yields_no_chunks_for_empty_files() {
let (chunks, oid) = chunk_source_whole(&PushSource::Bytes(Vec::new())).unwrap();
assert!(chunks.is_empty(), "an empty file must have no chunks");
assert_eq!(
oid, "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
"must still hash to git's empty-blob oid"
);
let (cdc_chunks, cdc_oid) =
chunk_source(&PushSource::Bytes(Vec::new()), 256, 1024, 4096).unwrap();
assert_eq!(chunks, cdc_chunks, "both passes must agree on empty input");
assert_eq!(oid, cdc_oid);
let data = b"hello".to_vec();
let (chunks, _) = chunk_source_whole(&PushSource::Bytes(data.clone())).unwrap();
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].1 as usize, data.len());
assert_eq!(chunks[0].0, <[u8; 32]>::from(Sha256::digest(&data)));
}
#[test]
fn stable_prefix_chunking_matches_a_full_rechunk() {
let (min, avg, max) = (256 * 1024, 1024 * 1024, 4 * 1024 * 1024);
let old: Vec<u8> = (0..5_000_000usize).map(|i| (i % 251) as u8).collect();
let appended: Vec<u8> = (0..2_000_000usize).map(|i| (i % 13) as u8).collect();
let mut grown = old.clone();
grown.extend_from_slice(&appended);
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("grown.bin");
std::fs::write(&path, &grown).unwrap();
let (old_chunks, _) = chunk_source(&PushSource::Bytes(old), min, avg, max).unwrap();
let stable = &old_chunks[..old_chunks.len() - 1];
let resumed = chunk_stable_prefix(&path, stable, min, avg, max).unwrap();
let (full, _) = chunk_source(&PushSource::Path(path.clone()), min, avg, max).unwrap();
assert_eq!(
resumed, full,
"prefix + resumed tail must equal a full re-chunk exactly"
);
assert_eq!(
resumed.iter().map(|(_, s)| *s as u64).sum::<u64>(),
grown.len() as u64,
"chunks must cover the grown file exactly"
);
}
}