use std::io::Read;
use std::path::{Component, Path, PathBuf};
use std::sync::Arc;
use ignore::WalkBuilder;
use reqwest::Method;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use crate::Traced;
use crate::error::SdkError;
use super::ArtifactStorageClient;
#[derive(Clone, Debug)]
pub struct PushFile {
pub repo_path: String,
pub source: PushSource,
pub mode: Option<u32>,
pub delete: bool,
}
#[non_exhaustive]
#[derive(Clone, Debug)]
pub enum PushSource {
Path(PathBuf),
Bytes(Vec<u8>),
KnownOid(String),
StablePrefix {
path: PathBuf,
stable_chunks: Vec<([u8; 32], u32)>,
},
}
#[derive(Clone, Debug)]
pub enum PushEvent {
Chunking {
files_done: usize,
files_total: usize,
bytes_hashed: u64,
},
Hashed {
files: usize,
chunks: usize,
bytes: u64,
},
Negotiated { missing: usize, total: usize },
UploadedBatch { chunks: usize, bytes: u64 },
Committing { files: usize },
CommitDetached { job_id: String },
CommitProgress {
phase: String,
done: u64,
total: u64,
},
Committed { commit: String, ref_name: String },
}
pub type PushProgress = Arc<dyn Fn(PushEvent) + Send + Sync>;
#[derive(Clone)]
pub struct PushOptions {
pub branch: String,
pub message: String,
pub base: Option<String>,
pub expect_oid: Option<String>,
pub upload_batch_bytes: usize,
pub progress: Option<PushProgress>,
pub workspace_snapshot: Option<String>,
pub collect_file_chunks: bool,
pub idempotency_key: Option<String>,
pub on_prepared: Option<PreparedHook>,
pub path_prefix: Option<String>,
}
pub type PreparedHook = Arc<dyn Fn(&[PreparedPush]) + Send + Sync>;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct PreparedPush {
pub path: String,
pub oid: Option<String>,
pub mode: Option<u32>,
pub delete: bool,
}
impl Default for PushOptions {
fn default() -> Self {
PushOptions {
branch: "main".to_string(),
message: String::new(),
base: None,
expect_oid: None,
upload_batch_bytes: 48 * 1024 * 1024,
progress: None,
workspace_snapshot: None,
collect_file_chunks: false,
idempotency_key: None,
on_prepared: None,
path_prefix: None,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct WorkspaceCheckpointOptions {
pub workspace_id: String,
pub generation: u64,
pub checkpoint_id: String,
pub base_ref: Option<String>,
pub publish_target: Option<String>,
pub mount_id: Option<String>,
pub mounted_on: Option<String>,
}
fn canonical_repo_path(path: &str, what: &str) -> Result<(), SdkError> {
if path.is_empty()
|| path.starts_with('/')
|| path.ends_with('/')
|| path.split('/').any(|component| {
component.is_empty()
|| component == "."
|| component == ".."
|| component.as_bytes().contains(&0)
})
{
return Err(SdkError::ClientError(format!(
"{what} must be a canonical repository-relative path, got {path:?}"
)));
}
Ok(())
}
fn encoded_repo_path(path: &str, what: &str) -> Result<String, SdkError> {
canonical_repo_path(path, what)?;
Ok(path
.split('/')
.map(super::encode_path_segment)
.collect::<Vec<_>>()
.join("/"))
}
fn is_hex_oid(value: &str) -> bool {
value.len() == 40 && value.bytes().all(|byte| byte.is_ascii_hexdigit())
}
fn is_lower_hex_oid(value: &str) -> bool {
value.len() == 40
&& value
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
}
fn apply_path_prefix(files: &mut [PushFile], prefix: Option<&str>) -> Result<(), SdkError> {
let Some(prefix) = prefix else {
return Ok(());
};
canonical_repo_path(prefix, "path prefix")?;
for file in files {
canonical_repo_path(&file.repo_path, "push path")?;
file.repo_path = format!("{prefix}/{}", file.repo_path);
}
Ok(())
}
pub type FileChunk = ([u8; 32], u32);
pub type FileChunks = Vec<FileChunk>;
pub type PushFileChunks = Vec<(String, FileChunks)>;
#[non_exhaustive]
#[derive(Clone, Debug, Serialize)]
pub struct PushReport {
pub commit: String,
pub tree: String,
pub ref_name: String,
pub created: bool,
pub files: usize,
pub bytes_total: u64,
pub chunks_total: usize,
pub chunks_uploaded: usize,
pub bytes_uploaded: u64,
pub file_blob_oids: Vec<(String, String)>,
#[serde(skip_serializing)]
pub file_chunks: PushFileChunks,
#[serde(skip_serializing_if = "Option::is_none")]
checkpoint: Option<WorkspaceCheckpointResponse>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkspaceCheckpointResponse {
pub format_ver: u16,
pub workspace_id: String,
pub ref_name: String,
pub generation: u64,
pub checkpoint_id: String,
pub base: String,
pub snapshot_tip: String,
pub tree: String,
pub created_at_ms: u64,
pub updated_at_ms: u64,
pub materialized_generation: Option<u64>,
}
#[non_exhaustive]
#[derive(Clone, Debug, Serialize)]
pub struct WorkspaceCheckpointPushReport {
pub checkpoint: WorkspaceCheckpointResponse,
pub files: usize,
pub bytes_total: u64,
pub chunks_total: usize,
pub chunks_uploaded: usize,
pub bytes_uploaded: u64,
pub file_blob_oids: Vec<(String, String)>,
#[serde(skip_serializing)]
pub file_chunks: PushFileChunks,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkspaceCheckpointChange {
pub path: String,
pub change: String,
pub mode: u32,
pub size: Option<u64>,
pub oid: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkspaceCheckpointChangesPage {
pub generation: u64,
pub checkpoint_id: String,
pub snapshot_tip: String,
pub tree: String,
pub materialized_generation: Option<u64>,
pub entries: Vec<WorkspaceCheckpointChange>,
pub truncated: bool,
pub next_after: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct MaterializeWorkspaceCheckpointRequest {
pub format_ver: u16,
pub generation: u64,
pub message: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub author: Option<super::merge::Signature>,
#[serde(skip_serializing_if = "Option::is_none")]
pub committer: Option<super::merge::Signature>,
#[serde(skip_serializing_if = "Option::is_none")]
pub mount_id: Option<String>,
pub publish_configured: bool,
}
impl MaterializeWorkspaceCheckpointRequest {
pub fn new(generation: u64, message: impl Into<String>) -> Self {
Self {
format_ver: 1,
generation,
message: message.into(),
author: None,
committer: None,
mount_id: None,
publish_configured: true,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum WorkspaceCheckpointPublishStatus {
NotConfigured,
Published,
Conflict,
Failed,
#[serde(other)]
Unknown,
}
impl WorkspaceCheckpointPublishStatus {
pub fn is_success(&self) -> bool {
matches!(self, Self::NotConfigured | Self::Published)
}
pub fn as_str(&self) -> &'static str {
match self {
Self::NotConfigured => "not_configured",
Self::Published => "published",
Self::Conflict => "conflict",
Self::Failed => "failed",
Self::Unknown => "unknown",
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct MaterializeWorkspaceCheckpointResponse {
pub generation: u64,
pub commit: String,
pub tree: String,
pub ref_name: String,
pub parent: Option<String>,
pub created: bool,
pub publish_status: WorkspaceCheckpointPublishStatus,
pub published_commit: Option<String>,
pub publish_error: Option<String>,
}
#[derive(Deserialize)]
struct IngestSessionWire {
session_id: String,
cdc_min_bytes: usize,
cdc_avg_bytes: usize,
cdc_max_bytes: usize,
max_hashes_per_query: usize,
#[allow(dead_code)]
max_chunk_bytes: usize,
#[serde(default)]
features: Vec<String>,
#[serde(default)]
small_file_max_bytes: u64,
#[serde(default)]
max_batch_files: usize,
}
#[derive(Serialize)]
struct MissingWire<'a> {
hashes: &'a [String],
}
#[derive(Deserialize)]
struct MissingRespWire {
missing: Vec<String>,
}
#[derive(Deserialize)]
struct IngestStagingWire {
pack_id: String,
url: Option<String>,
}
#[derive(Deserialize)]
struct BatchUploadRespWire {
files: Vec<BatchFileRespWire>,
}
#[derive(Deserialize)]
struct BatchFileRespWire {
token: String,
oid: String,
#[allow(dead_code)]
#[serde(default)]
deduplicated: bool,
}
#[derive(Serialize)]
struct StagedEntryWire {
hash: String,
offset: u64,
length: u32,
size: u32,
}
#[derive(Serialize)]
struct StagedRegisterWire {
pack_id: String,
entries: Vec<StagedEntryWire>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct CommitJobReadBack {
pub done: u64,
pub total: u64,
}
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
pub struct CommitJobError {
#[serde(default)]
pub kind: String,
#[serde(default)]
pub message: String,
#[serde(default)]
pub retryable: bool,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct CommitJobStatus {
pub job_id: String,
pub state: String,
#[serde(default)]
pub generation: Option<u64>,
#[serde(default)]
pub poll_url: Option<String>,
#[serde(default)]
pub phase: Option<String>,
#[serde(default)]
pub read_back: Option<CommitJobReadBack>,
#[serde(default)]
pub commit: Option<String>,
#[serde(default)]
pub tree: Option<String>,
#[serde(default)]
pub ref_name: Option<String>,
#[serde(default)]
pub parent: Option<String>,
#[serde(default)]
pub created: Option<bool>,
#[serde(default)]
pub error: Option<CommitJobError>,
}
const STAGED_ZSTD_LEVEL: i32 = 3;
const JOB_LONG_POLL_TIMEOUT_MS: u64 = 25_000;
#[derive(Serialize)]
struct CommitFileWire {
path: String,
#[serde(skip_serializing_if = "Vec::is_empty")]
chunks: Vec<CommitChunkWire>,
#[serde(skip_serializing_if = "Option::is_none")]
content: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
file_token: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
oid: Option<String>,
#[serde(skip_serializing_if = "std::ops::Not::not")]
delete: bool,
#[serde(skip_serializing_if = "Option::is_none")]
mode: Option<String>,
}
#[derive(Serialize)]
struct CommitChunkWire {
hash: String,
size: u32,
}
#[derive(Serialize)]
struct CommitWire {
#[serde(skip_serializing_if = "Option::is_none")]
branch: Option<String>,
message: String,
session_id: String,
files: Vec<CommitFileWire>,
#[serde(skip_serializing_if = "Option::is_none")]
base: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
expect_oid: Option<String>,
}
#[derive(Serialize)]
struct WorkspaceCheckpointWire<'a> {
format_ver: u16,
generation: u64,
checkpoint_id: &'a str,
base: &'a str,
#[serde(skip_serializing_if = "Option::is_none")]
base_ref: Option<&'a str>,
#[serde(skip_serializing_if = "Option::is_none")]
publish_target: Option<&'a str>,
#[serde(skip_serializing_if = "Option::is_none")]
mount_id: Option<&'a str>,
#[serde(skip_serializing_if = "Option::is_none")]
mounted_on: Option<&'a str>,
session_id: &'a str,
files: &'a [CommitFileWire],
}
#[derive(Clone, Debug, Deserialize)]
pub struct CommitByReferenceResponse {
pub commit: String,
pub tree: String,
pub ref_name: String,
pub parent: Option<String>,
pub created: bool,
}
struct ChunkedFile {
repo_path: String,
source: PushSource,
chunks: FileChunks,
mode: Option<u32>,
delete: bool,
known_oid: bool,
blob_oid: String,
cdc: bool,
}
fn chunk_source(
source: &PushSource,
min: usize,
avg: usize,
max: usize,
) -> Result<(FileChunks, String), SdkError> {
let len: u64 = match source {
PushSource::Path(p) => std::fs::metadata(p).map_err(io_err)?.len(),
PushSource::Bytes(b) => b.len() as u64,
PushSource::KnownOid(_) | PushSource::StablePrefix { .. } => {
return Err(SdkError::ClientError(
"source kind carries no full byte stream to chunk".to_string(),
));
}
};
let reader: Box<dyn Read + Send + '_> = match source {
PushSource::Path(p) => Box::new(std::fs::File::open(p).map_err(io_err)?),
PushSource::Bytes(b) => Box::new(std::io::Cursor::new(b.as_slice())),
PushSource::KnownOid(_) | PushSource::StablePrefix { .. } => {
unreachable!("guarded above")
}
};
let mut blob_hasher = BlobOidHasher::new(len);
let mut out = Vec::new();
for chunk in fastcdc::v2020::StreamCDC::new(reader, min as u32, avg as u32, max as u32) {
let chunk = chunk.map_err(|e| SdkError::ClientError(format!("chunking failed: {e}")))?;
let hash: [u8; 32] = Sha256::digest(&chunk.data).into();
blob_hasher.update(&chunk.data);
out.push((hash, chunk.data.len() as u32));
}
Ok((out, blob_hasher.finalize_hex()))
}
fn chunk_stable_prefix(
path: &Path,
stable_chunks: &[FileChunk],
min: usize,
avg: usize,
max: usize,
) -> Result<FileChunks, SdkError> {
use std::io::Seek as _;
let stable_len: u64 = stable_chunks.iter().map(|(_, s)| *s as u64).sum();
let mut file = std::fs::File::open(path).map_err(io_err)?;
file.seek(std::io::SeekFrom::Start(stable_len))
.map_err(io_err)?;
let mut out = stable_chunks.to_vec();
for chunk in fastcdc::v2020::StreamCDC::new(file, min as u32, avg as u32, max as u32) {
let chunk = chunk.map_err(|e| SdkError::ClientError(format!("chunking failed: {e}")))?;
let hash: [u8; 32] = Sha256::digest(&chunk.data).into();
out.push((hash, chunk.data.len() as u32));
}
Ok(out)
}
fn io_err(e: std::io::Error) -> SdkError {
SdkError::Io(e)
}
pub struct BlobOidHasher(sha1::Sha1);
impl BlobOidHasher {
pub fn new(len: u64) -> BlobOidHasher {
use sha1::Digest as _;
let mut h = sha1::Sha1::new();
h.update(format!("blob {len}\0").as_bytes());
BlobOidHasher(h)
}
pub fn update(&mut self, data: &[u8]) {
use sha1::Digest as _;
self.0.update(data);
}
pub fn finalize_hex(self) -> String {
use sha1::Digest as _;
hex::encode(self.0.finalize())
}
}
const TRANSIENT_RETRIES: usize = 3;
fn is_transient(e: &SdkError) -> bool {
match e {
SdkError::ServerError { status, .. } => status.is_server_error() || status.as_u16() == 429,
SdkError::Http(_) | SdkError::Middleware(_) => true,
_ => false,
}
}
pub(super) async fn with_transient_retries<T, F, Fut>(mut op: F) -> Result<T, SdkError>
where
F: FnMut() -> Fut,
Fut: std::future::Future<Output = Result<T, SdkError>>,
{
let mut delay = std::time::Duration::from_millis(250);
let mut attempt = 0usize;
loop {
match op().await {
Ok(v) => return Ok(v),
Err(e) if attempt < TRANSIENT_RETRIES && is_transient(&e) => {
attempt += 1;
let jitter = std::time::Duration::from_millis(u64::from(rand::random::<u8>()));
tokio::time::sleep(delay + jitter).await;
delay = (delay * 4).min(std::time::Duration::from_secs(4));
}
Err(e) => return Err(e),
}
}
}
fn chunk_source_whole(source: &PushSource) -> Result<(FileChunks, String), SdkError> {
let owned: Vec<u8>;
let data: &[u8] = match source {
PushSource::Path(p) => {
owned = std::fs::read(p).map_err(io_err)?;
&owned
}
PushSource::Bytes(b) => b,
PushSource::KnownOid(_) | PushSource::StablePrefix { .. } => {
return Err(SdkError::ClientError(
"source kind carries no full byte stream to hash".to_string(),
));
}
};
let mut blob = BlobOidHasher::new(data.len() as u64);
blob.update(data);
let chunks = if data.is_empty() {
Vec::new()
} else {
let hash: [u8; 32] = Sha256::digest(data).into();
vec![(hash, data.len() as u32)]
};
Ok((chunks, blob.finalize_hex()))
}
enum UploadReader<'a> {
File(std::fs::File),
Bytes(std::io::Cursor<&'a [u8]>),
}
impl<'a> UploadReader<'a> {
fn open(source: &'a PushSource) -> Result<UploadReader<'a>, SdkError> {
match source {
PushSource::Path(p) => Ok(UploadReader::File(std::fs::File::open(p).map_err(io_err)?)),
PushSource::StablePrefix { path, .. } => Ok(UploadReader::File(
std::fs::File::open(path).map_err(io_err)?,
)),
PushSource::Bytes(b) => Ok(UploadReader::Bytes(std::io::Cursor::new(b.as_slice()))),
PushSource::KnownOid(_) => Err(SdkError::ClientError(
"known-oid sources carry no bytes to upload".to_string(),
)),
}
}
fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), SdkError> {
match self {
UploadReader::File(f) => f.read_exact(buf).map_err(io_err),
UploadReader::Bytes(c) => c.read_exact(buf).map_err(io_err),
}
}
fn skip(&mut self, n: u64) -> Result<(), SdkError> {
use std::io::Seek as _;
match self {
UploadReader::File(f) => f.seek(std::io::SeekFrom::Current(n as i64)).map(|_| ()),
UploadReader::Bytes(c) => c.seek(std::io::SeekFrom::Current(n as i64)).map(|_| ()),
}
.map_err(io_err)
}
}
fn elect_tokened(
chunked: &[ChunkedFile],
missing: &std::collections::HashSet<[u8; 32]>,
small_max: u64,
) -> Vec<bool> {
chunked
.iter()
.map(|file| {
if file.delete || file.known_oid || file.chunks.is_empty() {
return false;
}
if matches!(file.source, PushSource::StablePrefix { .. }) {
return false;
}
let total: u64 = file.chunks.iter().map(|(_, s)| *s as u64).sum();
if small_max > 0 && total < small_max {
return true;
}
let missing_bytes: u64 = file
.chunks
.iter()
.filter(|(h, _)| missing.contains(h))
.map(|(_, s)| *s as u64)
.sum();
missing_bytes * 2 >= total
})
.collect()
}
fn dedup_needed_chunks(
chunked: &[ChunkedFile],
tokened: &[bool],
missing: &std::collections::HashSet<[u8; 32]>,
) -> std::collections::HashSet<[u8; 32]> {
chunked
.iter()
.zip(tokened)
.filter(|(file, tokened)| !**tokened && !file.delete && !file.known_oid)
.flat_map(|(file, _)| file.chunks.iter().map(|(h, _)| *h))
.filter(|h| missing.contains(h))
.collect()
}
fn collect_worktree_files(root: &Path, out: &mut Vec<PushFile>) -> Result<(), SdkError> {
let mut walker = WalkBuilder::new(root);
walker
.hidden(false)
.ignore(false)
.parents(false)
.require_git(false)
.git_ignore(true)
.git_exclude(true)
.git_global(true)
.follow_links(false)
.sort_by_file_path(|a, b| a.cmp(b))
.filter_entry(|entry| entry.file_name() != ".git");
for entry in walker.build() {
let entry = entry.map_err(|err| SdkError::ClientError(err.to_string()))?;
let path = entry.path();
let meta = std::fs::symlink_metadata(path)?;
if meta.is_dir() {
continue;
}
let repo_path = repo_path_for_worktree(root, path);
if repo_path.is_empty() {
continue;
}
let file_type = meta.file_type();
if file_type.is_symlink() {
let target = std::fs::read_link(path)?;
out.push(PushFile {
repo_path,
source: PushSource::Bytes(symlink_target_bytes(&target)),
mode: Some(0o120000),
delete: false,
});
} else if file_type.is_file() {
out.push(PushFile {
repo_path,
source: PushSource::Path(path.to_path_buf()),
mode: Some(regular_file_mode(&meta)),
delete: false,
});
}
}
Ok(())
}
fn regular_file_mode(meta: &std::fs::Metadata) -> u32 {
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
if meta.permissions().mode() & 0o111 != 0 {
0o100755
} else {
0o100644
}
}
#[cfg(not(unix))]
{
let _ = meta;
0o100644
}
}
fn symlink_target_bytes(target: &Path) -> Vec<u8> {
#[cfg(unix)]
{
use std::os::unix::ffi::OsStrExt;
target.as_os_str().as_bytes().to_vec()
}
#[cfg(not(unix))]
{
target.to_string_lossy().into_owned().into_bytes()
}
}
fn repo_path_for_worktree(root: &Path, path: &Path) -> String {
path.strip_prefix(root)
.unwrap_or(path)
.components()
.filter_map(|c| match c {
Component::Normal(part) => Some(part.to_string_lossy()),
_ => None,
})
.collect::<Vec<_>>()
.join("/")
}
impl ArtifactStorageClient {
#[allow(clippy::too_many_arguments)]
pub async fn push_worktree(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
root: impl AsRef<Path>,
opts: PushOptions,
) -> Result<Traced<PushReport>, SdkError> {
let root = root.as_ref();
let mut files = Vec::new();
collect_worktree_files(root, &mut files)?;
if files.is_empty() {
return Err(SdkError::ClientError(
"no files found under the Git worktree".to_string(),
));
}
self.push_files(project_id, repo, git_username, git_token, files, opts)
.await
}
#[allow(clippy::too_many_arguments)]
pub async fn push_files(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
files: Vec<PushFile>,
opts: PushOptions,
) -> Result<Traced<PushReport>, SdkError> {
self.push_files_inner(project_id, repo, git_username, git_token, files, opts, None)
.await
}
#[allow(clippy::too_many_arguments)]
async fn push_files_inner(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
mut files: Vec<PushFile>,
opts: PushOptions,
workspace_checkpoint: Option<WorkspaceCheckpointOptions>,
) -> Result<Traced<PushReport>, SdkError> {
apply_path_prefix(&mut files, opts.path_prefix.as_deref())?;
if opts.workspace_snapshot.is_some() && workspace_checkpoint.is_some() {
return Err(SdkError::ClientError(
"workspace_snapshot and workspace_checkpoint are mutually exclusive".into(),
));
}
if let Some(checkpoint) = workspace_checkpoint.as_ref() {
if !is_lower_hex_oid(&checkpoint.workspace_id) {
return Err(SdkError::ClientError(format!(
"workspace checkpoint id must be a 40-character lowercase hex id, got {:?}",
checkpoint.workspace_id
)));
}
if checkpoint.generation == 0 {
return Err(SdkError::ClientError(
"workspace checkpoint generation must be greater than zero".into(),
));
}
if checkpoint.checkpoint_id.is_empty() || checkpoint.checkpoint_id.len() > 128 {
return Err(SdkError::ClientError(
"workspace checkpoint idempotency id must contain 1..128 bytes".into(),
));
}
if !opts.base.as_deref().is_some_and(is_hex_oid) {
return Err(SdkError::ClientError(
"workspace checkpoint requires a 40-hex base commit".into(),
));
}
}
let emit = |ev: PushEvent| {
if let Some(p) = &opts.progress {
p(ev)
}
};
let session: IngestSessionWire = with_transient_retries(|| async {
let (req, _trace) = self.git_request(
Method::POST,
project_id,
repo,
Some("ingest/sessions"),
git_username,
git_token,
)?;
expect_json(req.send().await?).await
})
.await?;
let (min, avg, max) = (
session.cdc_min_bytes,
session.cdc_avg_bytes,
session.cdc_max_bytes,
);
let small_max = session.small_file_max_bytes;
let hash_parallelism = std::thread::available_parallelism()
.map(|n| n.get().min(8))
.unwrap_or(4);
let chunked: Vec<ChunkedFile> = {
use futures::StreamExt as _;
let files_total = files.len();
let mut stream = futures::stream::iter(files.into_iter().map(|f| async move {
if f.delete {
return Ok::<ChunkedFile, SdkError>(ChunkedFile {
repo_path: f.repo_path,
source: f.source,
chunks: Vec::new(),
mode: f.mode,
delete: true,
known_oid: false,
blob_oid: String::new(),
cdc: false,
});
}
if let PushSource::KnownOid(oid) = &f.source {
let oid = oid.to_ascii_lowercase();
if oid.len() != 40 || !oid.bytes().all(|b| b.is_ascii_hexdigit()) {
return Err(SdkError::ClientError(format!(
"known oid for {:?} is not a 40-hex git oid: {oid:?}",
f.repo_path
)));
}
return Ok(ChunkedFile {
repo_path: f.repo_path,
blob_oid: oid,
source: f.source,
chunks: Vec::new(),
mode: f.mode,
delete: false,
known_oid: true,
cdc: false,
});
}
let mut f = f;
let source_len: u64 = match &f.source {
PushSource::Path(p) => std::fs::metadata(p).map_err(io_err)?.len(),
PushSource::Bytes(b) => b.len() as u64,
PushSource::StablePrefix { path, .. } => {
std::fs::metadata(path).map_err(io_err)?.len()
}
PushSource::KnownOid(_) => unreachable!("handled above"),
};
let whole = small_max > 0 && source_len < small_max;
if let PushSource::StablePrefix {
path,
stable_chunks,
} = &f.source
{
let stable_len: u64 = stable_chunks.iter().map(|(_, s)| *s as u64).sum();
if whole || stable_chunks.is_empty() || stable_len > source_len {
f.source = PushSource::Path(path.clone());
}
}
tokio::task::spawn_blocking(move || {
let (chunks, blob_oid, cdc) = match &f.source {
PushSource::StablePrefix {
path,
stable_chunks,
} => (
chunk_stable_prefix(path, stable_chunks, min, avg, max)?,
String::new(),
true,
),
source if whole => {
let (chunks, oid) = chunk_source_whole(source)?;
(chunks, oid, false)
}
source => {
let (chunks, oid) = chunk_source(source, min, avg, max)?;
(chunks, oid, true)
}
};
Ok(ChunkedFile {
repo_path: f.repo_path,
source: f.source,
chunks,
mode: f.mode,
delete: false,
known_oid: false,
blob_oid,
cdc,
})
})
.await
.map_err(|e| SdkError::ClientError(format!("chunking task failed: {e}")))?
}))
.buffered(hash_parallelism);
let mut out: Vec<ChunkedFile> = Vec::with_capacity(files_total);
let mut bytes_hashed = 0u64;
let mut last_emit = std::time::Instant::now();
while let Some(result) = stream.next().await {
let file: ChunkedFile = result?;
bytes_hashed += file.chunks.iter().map(|(_, s)| *s as u64).sum::<u64>();
out.push(file);
if last_emit.elapsed() >= std::time::Duration::from_millis(100) {
last_emit = std::time::Instant::now();
emit(PushEvent::Chunking {
files_done: out.len(),
files_total,
bytes_hashed,
});
}
}
out
};
if chunked.iter().any(|f| f.known_oid) && !session.features.iter().any(|f| f == "oid_files")
{
return Err(SdkError::ClientError(
"this artifact-storage server does not support known-oid file references \
(`oid_files` capability missing); re-push the content instead"
.to_string(),
));
}
if let Some(hook) = &opts.on_prepared {
let prepared: Vec<PreparedPush> = chunked
.iter()
.map(|f| PreparedPush {
path: f.repo_path.clone(),
oid: (!f.delete && !f.blob_oid.is_empty()).then(|| f.blob_oid.clone()),
mode: f.mode,
delete: f.delete,
})
.collect();
hook(&prepared);
}
let total_chunks: usize = chunked.iter().map(|f| f.chunks.len()).sum();
let total_bytes: u64 = chunked
.iter()
.flat_map(|f| f.chunks.iter().map(|(_, s)| *s as u64))
.sum();
emit(PushEvent::Hashed {
files: chunked.len(),
chunks: total_chunks,
bytes: total_bytes,
});
let is_small = |f: &ChunkedFile| -> bool {
small_max > 0
&& f.chunks.iter().map(|(_, s)| *s as u64).sum::<u64>() < small_max
&& !f.delete
&& !f.known_oid
&& !f.chunks.is_empty()
};
let mut distinct: Vec<[u8; 32]> = chunked
.iter()
.filter(|f| !is_small(f))
.flat_map(|f| f.chunks.iter().map(|(h, _)| *h))
.collect();
distinct.sort_unstable();
distinct.dedup();
let missing = self
.negotiate_missing(
project_id,
repo,
git_username,
git_token,
&session,
&distinct,
)
.await?;
emit(PushEvent::Negotiated {
missing: missing.len(),
total: distinct.len(),
});
let (mut uploaded_chunks, mut uploaded_bytes) = (0usize, 0u64);
let mut file_tokens: Vec<Option<String>> = vec![None; chunked.len()];
let tokened = elect_tokened(&chunked, &missing, small_max);
{
let mut token_by_oid: std::collections::HashMap<&str, String> =
std::collections::HashMap::new();
let mut owners: Vec<usize> = Vec::new();
for (i, file) in chunked.iter().enumerate() {
if !tokened[i] {
continue;
}
let token = token_by_oid
.entry(file.blob_oid.as_str())
.or_insert_with(|| {
owners.push(i);
format!("f{i}-{}", hex::encode(&file.chunks[0].0[..8]))
});
file_tokens[i] = Some(token.clone());
}
use futures::StreamExt as _;
const TOKENED_UPLOADS_IN_FLIGHT: usize = 4;
let batch_capable = session.features.iter().any(|f| f == "batch_files")
&& session.small_file_max_bytes > 0
&& session.max_batch_files > 0;
let mut batch_owners: Vec<usize> = Vec::new();
let mut single_owners: Vec<usize> = Vec::new();
for i in owners {
let total: u64 = chunked[i].chunks.iter().map(|(_, s)| *s as u64).sum();
if batch_capable && total < session.small_file_max_bytes {
batch_owners.push(i);
} else {
single_owners.push(i);
}
}
const BATCH_GROUP_BYTES: usize = 16 * 1024 * 1024;
let mut groups: Vec<Vec<usize>> = Vec::new();
{
let mut cur: Vec<usize> = Vec::new();
let mut cur_bytes = 0usize;
for &i in &batch_owners {
let file = &chunked[i];
let record_bytes: usize = 14
+ file_tokens[i].as_ref().map(String::len).unwrap_or(0)
+ file
.chunks
.iter()
.map(|(_, s)| 36 + *s as usize)
.sum::<usize>();
if !cur.is_empty()
&& (cur_bytes + record_bytes > BATCH_GROUP_BYTES
|| cur.len() >= session.max_batch_files)
{
groups.push(std::mem::take(&mut cur));
cur_bytes = 0;
}
cur.push(i);
cur_bytes += record_bytes;
}
if !cur.is_empty() {
groups.push(cur);
}
}
let mut batch_results = futures::stream::iter(groups.into_iter().map(|group| {
let session_id = session.session_id.as_str();
let file_tokens = &file_tokens;
let chunked = &chunked;
async move {
let mut body = Vec::new();
let mut group_chunks = 0usize;
let mut group_bytes = 0u64;
let mut expected: Vec<(String, String)> = Vec::new();
for &i in &group {
let file = &chunked[i];
let token = file_tokens[i].clone().expect("owner has a token");
let total: u64 = file.chunks.iter().map(|(_, s)| *s as u64).sum();
body.extend_from_slice(&(token.len() as u16).to_be_bytes());
body.extend_from_slice(&total.to_be_bytes());
body.extend_from_slice(&(file.chunks.len() as u32).to_be_bytes());
body.extend_from_slice(token.as_bytes());
let mut reader: Box<dyn Read + Send + '_> = match &file.source {
PushSource::Path(p) => {
Box::new(std::fs::File::open(p).map_err(io_err)?)
}
PushSource::Bytes(b) => Box::new(std::io::Cursor::new(b.as_slice())),
PushSource::KnownOid(_) | PushSource::StablePrefix { .. } => {
unreachable!("never tokened")
}
};
for (hash, size) in &file.chunks {
let mut data = vec![0u8; *size as usize];
reader.read_exact(&mut data).map_err(io_err)?;
body.extend_from_slice(hash);
body.extend_from_slice(&(data.len() as u32).to_be_bytes());
body.extend_from_slice(&data);
group_chunks += 1;
group_bytes += data.len() as u64;
}
expected.push((token, file.blob_oid.clone()));
}
let resp = self
.put_batch_files(
project_id,
repo,
git_username,
git_token,
session_id,
body,
)
.await?;
let by_token: std::collections::HashMap<&str, &str> = resp
.files
.iter()
.map(|f| (f.token.as_str(), f.oid.as_str()))
.collect();
for (token, oid) in &expected {
match by_token.get(token.as_str()) {
Some(server) if server == oid => {}
Some(server) => {
return Err(SdkError::ClientError(format!(
"server hashed {token:?} to {server} but the client computed {oid}; was the file modified mid-push?"
)));
}
None => {
return Err(SdkError::ClientError(format!(
"batch upload response is missing token {token:?}"
)));
}
}
}
Ok::<(usize, u64), SdkError>((group_chunks, group_bytes))
}
}))
.buffer_unordered(TOKENED_UPLOADS_IN_FLIGHT);
while let Some(done) = batch_results.next().await {
let (chunks, bytes) = done?;
uploaded_chunks += chunks;
uploaded_bytes += bytes;
emit(PushEvent::UploadedBatch {
chunks: uploaded_chunks,
bytes: uploaded_bytes,
});
}
drop(batch_results);
let mut results = futures::stream::iter(single_owners.into_iter().map(|i| {
let file = &chunked[i];
let token = file_tokens[i].clone().expect("owner has a token");
let session_id = session.session_id.as_str();
let batch_bytes = opts.upload_batch_bytes;
async move {
let total: u64 = file.chunks.iter().map(|(_, s)| *s as u64).sum();
let mut reader: Box<dyn Read + Send + '_> = match &file.source {
PushSource::Path(p) => Box::new(std::fs::File::open(p).map_err(io_err)?),
PushSource::Bytes(b) => Box::new(std::io::Cursor::new(b.as_slice())),
PushSource::KnownOid(_) | PushSource::StablePrefix { .. } => {
unreachable!("never tokened")
}
};
let (mut sent_chunks, mut sent_bytes) = (0usize, 0u64);
let mut offset = 0u64;
let mut frame = Vec::with_capacity(batch_bytes + 64);
for (hash, size) in &file.chunks {
let mut data = vec![0u8; *size as usize];
reader.read_exact(&mut data).map_err(io_err)?;
frame.extend_from_slice(hash);
frame.extend_from_slice(&(data.len() as u32).to_be_bytes());
frame.extend_from_slice(&data);
sent_chunks += 1;
sent_bytes += data.len() as u64;
if frame.len() >= batch_bytes {
let sent: u64 = frame_payload_bytes(&frame);
self.put_chunk_frame(
project_id,
repo,
git_username,
git_token,
session_id,
Some((&token, total, offset)),
std::mem::take(&mut frame),
)
.await?;
offset += sent;
}
}
if !frame.is_empty() {
self.put_chunk_frame(
project_id,
repo,
git_username,
git_token,
session_id,
Some((&token, total, offset)),
frame,
)
.await?;
}
Ok::<(usize, u64), SdkError>((sent_chunks, sent_bytes))
}
}))
.buffer_unordered(TOKENED_UPLOADS_IN_FLIGHT);
while let Some(done) = results.next().await {
let (chunks, bytes) = done?;
uploaded_chunks += chunks;
uploaded_bytes += bytes;
emit(PushEvent::UploadedBatch {
chunks: uploaded_chunks,
bytes: uploaded_bytes,
});
}
}
let mut to_upload = dedup_needed_chunks(&chunked, &tokened, &missing);
const DEDUP_UPLOADS_IN_FLIGHT: usize = 3;
let staging_target = if to_upload.is_empty() {
None
} else {
match self
.ingest_staging_target(
project_id,
repo,
git_username,
git_token,
&session.session_id,
)
.await
{
Ok(probe) => probe.url.is_some().then_some(probe),
Err(SdkError::ServerError { status, .. })
if status.as_u16() == 404 || status.as_u16() == 405 =>
{
None
}
Err(e) => return Err(e),
}
};
if let Some(first_target) = staging_target {
use futures::StreamExt as _;
let mut inflight = futures::stream::FuturesUnordered::new();
let mut next_target = Some(first_target);
let mut pack: Vec<u8> = Vec::with_capacity(opts.upload_batch_bytes + 256 * 1024);
let mut entries: Vec<StagedEntryWire> = Vec::new();
for file in &chunked {
if to_upload.is_empty() {
break;
}
if file.chunks.is_empty() {
continue; }
let mut reader = UploadReader::open(&file.source)?;
for (hash, size) in &file.chunks {
if !to_upload.remove(hash) {
reader.skip(*size as u64)?;
continue;
}
let mut data = vec![0u8; *size as usize];
reader.read_exact(&mut data)?;
let zframe = zstd::encode_all(&data[..], STAGED_ZSTD_LEVEL)
.map_err(|e| SdkError::ClientError(format!("zstd encode: {e}")))?;
entries.push(StagedEntryWire {
hash: hex_lower(hash),
offset: pack.len() as u64,
length: zframe.len() as u32,
size: *size,
});
pack.extend_from_slice(&zframe);
uploaded_chunks += 1;
uploaded_bytes += data.len() as u64;
if pack.len() >= opts.upload_batch_bytes {
let target = match next_target.take() {
Some(t) => t,
None => {
self.ingest_staging_target(
project_id,
repo,
git_username,
git_token,
&session.session_id,
)
.await?
}
};
inflight.push(self.upload_staged_pack(
project_id,
repo,
git_username,
git_token,
&session.session_id,
target,
std::mem::take(&mut pack),
std::mem::take(&mut entries),
));
emit(PushEvent::UploadedBatch {
chunks: uploaded_chunks,
bytes: uploaded_bytes,
});
if inflight.len() >= DEDUP_UPLOADS_IN_FLIGHT {
inflight.next().await.expect("inflight upload present")?;
}
}
}
}
if !pack.is_empty() {
let target = match next_target.take() {
Some(t) => t,
None => {
self.ingest_staging_target(
project_id,
repo,
git_username,
git_token,
&session.session_id,
)
.await?
}
};
inflight.push(self.upload_staged_pack(
project_id,
repo,
git_username,
git_token,
&session.session_id,
target,
pack,
entries,
));
emit(PushEvent::UploadedBatch {
chunks: uploaded_chunks,
bytes: uploaded_bytes,
});
}
while let Some(done) = inflight.next().await {
done?;
}
} else {
use futures::StreamExt as _;
let mut inflight = futures::stream::FuturesUnordered::new();
let mut frame = Vec::with_capacity(opts.upload_batch_bytes + 64);
for file in &chunked {
if to_upload.is_empty() {
break;
}
if file.chunks.is_empty() {
continue; }
let mut reader = UploadReader::open(&file.source)?;
for (hash, size) in &file.chunks {
if !to_upload.remove(hash) {
reader.skip(*size as u64)?;
continue;
}
let mut data = vec![0u8; *size as usize];
reader.read_exact(&mut data)?;
frame.extend_from_slice(hash);
frame.extend_from_slice(&(data.len() as u32).to_be_bytes());
frame.extend_from_slice(&data);
uploaded_chunks += 1;
uploaded_bytes += data.len() as u64;
if frame.len() >= opts.upload_batch_bytes {
inflight.push(self.put_chunk_frame(
project_id,
repo,
git_username,
git_token,
&session.session_id,
None,
std::mem::take(&mut frame),
));
emit(PushEvent::UploadedBatch {
chunks: uploaded_chunks,
bytes: uploaded_bytes,
});
if inflight.len() >= DEDUP_UPLOADS_IN_FLIGHT {
inflight.next().await.expect("inflight upload present")?;
}
}
}
}
if !frame.is_empty() {
inflight.push(self.put_chunk_frame(
project_id,
repo,
git_username,
git_token,
&session.session_id,
None,
frame,
));
emit(PushEvent::UploadedBatch {
chunks: uploaded_chunks,
bytes: uploaded_bytes,
});
}
while let Some(done) = inflight.next().await {
done?;
}
}
let token_commits = session.features.iter().any(|f| f == "token_commits");
let commit_files: Vec<CommitFileWire> = chunked
.iter()
.enumerate()
.map(|(i, f)| {
let token_only = token_commits && file_tokens[i].is_some() && is_small(f);
CommitFileWire {
path: f.repo_path.clone(),
chunks: if token_only {
Vec::new()
} else {
f.chunks
.iter()
.map(|(h, s)| CommitChunkWire {
hash: hex_lower(h),
size: *s,
})
.collect()
},
content: (!f.delete && !f.known_oid && !token_only && f.chunks.is_empty())
.then(String::new),
file_token: file_tokens[i].clone(),
oid: (f.known_oid || token_only).then(|| f.blob_oid.clone()),
delete: f.delete,
mode: f.mode.map(|m| format!("{m:o}")),
}
})
.collect();
if let Some(checkpoint) = workspace_checkpoint.as_ref() {
let base = opts
.base
.as_deref()
.expect("checkpoint base validated above");
let body = WorkspaceCheckpointWire {
format_ver: 1,
generation: checkpoint.generation,
checkpoint_id: &checkpoint.checkpoint_id,
base,
base_ref: checkpoint.base_ref.as_deref(),
publish_target: checkpoint.publish_target.as_deref(),
mount_id: checkpoint.mount_id.as_deref(),
mounted_on: checkpoint.mounted_on.as_deref(),
session_id: &session.session_id,
files: &commit_files,
};
let suffix = format!(
"workspaces/{}/checkpoint",
super::encode_path_segment(&checkpoint.workspace_id)
);
let (checkpoint_response, trace_id) = with_transient_retries(|| async {
let (request, trace_id) = self.git_request(
Method::PUT,
project_id,
repo,
Some(&suffix),
git_username,
git_token,
)?;
let response = request
.timeout(std::time::Duration::from_secs(60))
.json(&body)
.send()
.await?;
let checkpoint: WorkspaceCheckpointResponse = expect_json(response).await?;
Ok((checkpoint, trace_id))
})
.await?;
return Ok(Traced::new(
trace_id,
PushReport {
commit: checkpoint_response.snapshot_tip.clone(),
tree: checkpoint_response.tree.clone(),
ref_name: checkpoint_response.ref_name.clone(),
created: false,
files: chunked.len(),
bytes_total: total_bytes,
chunks_total: distinct.len(),
chunks_uploaded: uploaded_chunks,
bytes_uploaded: uploaded_bytes,
file_chunks: if opts.collect_file_chunks {
chunked
.iter()
.filter(|file| !file.delete && !file.known_oid && file.cdc)
.map(|file| (file.repo_path.clone(), file.chunks.clone()))
.collect()
} else {
Vec::new()
},
file_blob_oids: chunked
.into_iter()
.map(|file| (file.repo_path, file.blob_oid))
.collect(),
checkpoint: Some(checkpoint_response),
},
));
}
let (commit_suffix, branch) = match &opts.workspace_snapshot {
Some(ws_id) => (
format!("workspaces/{}/snapshots", super::encode_path_segment(ws_id)),
None,
),
None => ("commits".to_string(), Some(opts.branch.clone())),
};
let body = CommitWire {
branch,
message: opts.message.clone(),
session_id: session.session_id.clone(),
files: commit_files,
base: opts.base.clone(),
expect_oid: opts.expect_oid.clone(),
};
emit(PushEvent::Committing {
files: body.files.len(),
});
let idem_key = opts.idempotency_key.clone().unwrap_or_else(|| {
let bytes: [u8; 16] = rand::random();
bytes.iter().map(|b| format!("{b:02x}")).collect::<String>()
});
let (accepted, mut job, trace_id) = with_transient_retries(|| async {
let (req, trace_id) = self.git_request(
Method::POST,
project_id,
repo,
Some(&commit_suffix),
git_username,
git_token,
)?;
let submit = req
.header("x-commit-async", "1")
.header("idempotency-key", &idem_key)
.timeout(std::time::Duration::from_secs(60))
.json(&body)
.send()
.await?;
let accepted = submit.status().as_u16() == 202;
let job: CommitJobStatus = expect_json(submit).await?;
Ok((accepted, job, trace_id))
})
.await?;
if accepted {
let job_id = job.job_id.clone();
emit(PushEvent::CommitDetached {
job_id: job_id.clone(),
});
let mut delay = std::time::Duration::from_millis(500);
let mut last_gen: Option<u64> = job.generation;
loop {
job = with_transient_retries(|| async {
self.commit_job_get(
project_id,
repo,
git_username,
git_token,
&job_id,
last_gen,
)
.await
.map(Traced::into_inner)
})
.await?;
match job.state.as_str() {
"committed" | "failed" => break,
_ => {
let (done, total) = job
.read_back
.as_ref()
.map_or((0, 0), |rb| (rb.done, rb.total));
emit(PushEvent::CommitProgress {
phase: job.phase.clone().unwrap_or_else(|| job.state.clone()),
done,
total,
});
}
}
let advanced = match (last_gen, job.generation) {
(Some(prev), Some(cur)) => cur > prev,
(None, Some(_)) => true, _ => false,
};
if job.generation.is_some() {
last_gen = job.generation;
}
if advanced {
delay = std::time::Duration::from_millis(500);
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
} else {
tokio::time::sleep(delay).await;
delay = (delay * 2).min(std::time::Duration::from_secs(2));
}
}
}
if job.state == "failed" {
let err = job.error.unwrap_or_default();
return Err(SdkError::ClientError(format!(
"commit job failed ({}): {}{}",
err.kind,
err.message,
if err.retryable {
" (safe to retry: uploaded chunks are deduplicated)"
} else {
""
}
)));
}
let resp = CommitByReferenceResponse {
commit: job.commit.ok_or_else(|| {
SdkError::ClientError("committed job missing commit oid".to_string())
})?,
tree: job
.tree
.ok_or_else(|| SdkError::ClientError("committed job missing tree".to_string()))?,
ref_name: job.ref_name.ok_or_else(|| {
SdkError::ClientError("committed job missing ref name".to_string())
})?,
parent: job.parent,
created: job.created.unwrap_or(false),
};
emit(PushEvent::Committed {
commit: resp.commit.clone(),
ref_name: resp.ref_name.clone(),
});
Ok(Traced::new(
trace_id,
PushReport {
commit: resp.commit,
tree: resp.tree,
ref_name: resp.ref_name,
created: resp.created,
files: chunked.len(),
bytes_total: total_bytes,
chunks_total: distinct.len(),
chunks_uploaded: uploaded_chunks,
bytes_uploaded: uploaded_bytes,
file_chunks: if opts.collect_file_chunks {
chunked
.iter()
.filter(|f| !f.delete && !f.known_oid && f.cdc)
.map(|f| (f.repo_path.clone(), f.chunks.clone()))
.collect()
} else {
Vec::new()
},
file_blob_oids: chunked
.into_iter()
.map(|f| (f.repo_path, f.blob_oid))
.collect(),
checkpoint: None,
},
))
}
#[allow(clippy::too_many_arguments)]
pub async fn push_workspace_checkpoint(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
files: Vec<PushFile>,
opts: PushOptions,
target: WorkspaceCheckpointOptions,
) -> Result<Traced<WorkspaceCheckpointPushReport>, SdkError> {
if opts.workspace_snapshot.is_some() {
return Err(SdkError::ClientError(
"workspace checkpoint push cannot also target a snapshot commit".into(),
));
}
let report = self
.push_files_inner(
project_id,
repo,
git_username,
git_token,
files,
opts,
Some(target),
)
.await?;
let trace_id = report.trace_id.clone();
let report = report.into_inner();
let checkpoint = report.checkpoint.ok_or_else(|| {
SdkError::ClientError("checkpoint push returned a commit response".into())
})?;
Ok(Traced::new(
trace_id,
WorkspaceCheckpointPushReport {
checkpoint,
files: report.files,
bytes_total: report.bytes_total,
chunks_total: report.chunks_total,
chunks_uploaded: report.chunks_uploaded,
bytes_uploaded: report.bytes_uploaded,
file_blob_oids: report.file_blob_oids,
file_chunks: report.file_chunks,
},
))
}
pub async fn workspace_checkpoint(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
workspace_id: &str,
) -> Result<Traced<WorkspaceCheckpointResponse>, SdkError> {
let suffix = format!(
"workspaces/{}/checkpoint",
super::encode_path_segment(workspace_id)
);
let (response, trace_id) = with_transient_retries(|| async {
let (request, trace_id) = self.git_request(
Method::GET,
project_id,
repo,
Some(&suffix),
git_username,
git_token,
)?;
let response = expect_json(request.send().await?).await?;
Ok((response, trace_id))
})
.await?;
Ok(Traced::new(trace_id, response))
}
#[allow(clippy::too_many_arguments)]
pub async fn workspace_checkpoint_changes(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
workspace_id: &str,
after: Option<&str>,
limit: usize,
) -> Result<Traced<WorkspaceCheckpointChangesPage>, SdkError> {
let suffix = format!(
"workspaces/{}/checkpoint/changes",
super::encode_path_segment(workspace_id)
);
let mut query = vec![("limit", limit.clamp(1, 1_000).to_string())];
if let Some(after) = after {
query.push(("after", after.to_string()));
}
let (response, trace_id) = with_transient_retries(|| async {
let (request, trace_id) = self.git_request(
Method::GET,
project_id,
repo,
Some(&suffix),
git_username,
git_token,
)?;
let response = expect_json(request.query(&query).send().await?).await?;
Ok((response, trace_id))
})
.await?;
Ok(Traced::new(trace_id, response))
}
#[allow(clippy::too_many_arguments)]
pub async fn workspace_checkpoint_file_bytes(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
workspace_id: &str,
file_path: &str,
generation: u64,
checkpoint_id: &str,
tree: &str,
) -> Result<Traced<Vec<u8>>, SdkError> {
let encoded_file_path = encoded_repo_path(file_path, "checkpoint file path")?;
let suffix = format!(
"workspaces/{}/checkpoint/files/{encoded_file_path}",
super::encode_path_segment(workspace_id)
);
let query = [
("generation", generation.to_string()),
("checkpoint_id", checkpoint_id.to_string()),
("tree", tree.to_string()),
];
let (bytes, trace_id) = with_transient_retries(|| async {
let (request, trace_id) = self.git_request(
Method::GET,
project_id,
repo,
Some(&suffix),
git_username,
git_token,
)?;
let response = request.query(&query).send().await?;
let status = response.status();
if !status.is_success() {
return Err(SdkError::ServerError {
status,
message: response.text().await.unwrap_or_default(),
});
}
Ok((response.bytes().await?.to_vec(), trace_id))
})
.await?;
Ok(Traced::new(trace_id, bytes))
}
#[allow(clippy::too_many_arguments)]
pub async fn workspace_checkpoint_file_range(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
workspace_id: &str,
file_path: &str,
generation: u64,
checkpoint_id: &str,
tree: &str,
start: u64,
end_inclusive: u64,
) -> Result<Traced<Vec<u8>>, SdkError> {
let encoded_file_path = encoded_repo_path(file_path, "checkpoint file path")?;
if start > end_inclusive {
return Err(SdkError::ClientError(
"checkpoint file range start exceeds end".to_string(),
));
}
let suffix = format!(
"workspaces/{}/checkpoint/files/{encoded_file_path}",
super::encode_path_segment(workspace_id)
);
let query = [
("generation", generation.to_string()),
("checkpoint_id", checkpoint_id.to_string()),
("tree", tree.to_string()),
];
let expected_len = end_inclusive - start + 1;
let range_header = format!("bytes={start}-{end_inclusive}");
let (bytes, trace_id) = with_transient_retries(|| async {
let (request, trace_id) = self.git_request(
Method::GET,
project_id,
repo,
Some(&suffix),
git_username,
git_token,
)?;
let response = request
.query(&query)
.header(reqwest::header::RANGE, &range_header)
.send()
.await?;
let status = response.status();
if status != reqwest::StatusCode::PARTIAL_CONTENT {
if status.is_success() {
return Err(SdkError::ClientError(format!(
"checkpoint file range endpoint returned {status} instead of 206 Partial Content"
)));
}
return Err(SdkError::ServerError {
status,
message: response.text().await.unwrap_or_default(),
});
}
let expected_content_range = format!("bytes {start}-{end_inclusive}/");
let content_range_matches = response
.headers()
.get(reqwest::header::CONTENT_RANGE)
.and_then(|value| value.to_str().ok())
.is_some_and(|value| value.starts_with(&expected_content_range));
if !content_range_matches {
return Err(SdkError::ClientError(format!(
"checkpoint file range response did not bind bytes {start}-{end_inclusive}"
)));
}
let bytes = response.bytes().await?.to_vec();
if bytes.len() as u64 != expected_len {
return Err(SdkError::ClientError(format!(
"checkpoint file range returned {} bytes, expected {expected_len}",
bytes.len()
)));
}
Ok((bytes, trace_id))
})
.await?;
Ok(Traced::new(trace_id, bytes))
}
#[allow(clippy::too_many_arguments)]
pub async fn materialize_workspace_checkpoint(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
workspace_id: &str,
request: &MaterializeWorkspaceCheckpointRequest,
) -> Result<Traced<MaterializeWorkspaceCheckpointResponse>, SdkError> {
let suffix = format!(
"workspaces/{}/checkpoint/materialize",
super::encode_path_segment(workspace_id)
);
let (response, trace_id) = with_transient_retries(|| async {
let (http_request, trace_id) = self.git_request(
Method::POST,
project_id,
repo,
Some(&suffix),
git_username,
git_token,
)?;
let response = expect_json(http_request.json(request).send().await?).await?;
Ok((response, trace_id))
})
.await?;
Ok(Traced::new(trace_id, response))
}
async fn negotiate_missing(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
session: &IngestSessionWire,
distinct: &[[u8; 32]],
) -> Result<std::collections::HashSet<[u8; 32]>, SdkError> {
let mut missing = std::collections::HashSet::new();
for batch in distinct.chunks(session.max_hashes_per_query.max(1)) {
let hashes: Vec<String> = batch.iter().map(hex_lower).collect();
let resp: MissingRespWire = with_transient_retries(|| async {
let (req, _t) = self.git_request(
Method::POST,
project_id,
repo,
Some(&format!("ingest/sessions/{}/missing", session.session_id)),
git_username,
git_token,
)?;
expect_json(req.json(&MissingWire { hashes: &hashes }).send().await?).await
})
.await?;
for h in resp.missing {
let mut arr = [0u8; 32];
hex::decode_to_slice(&h, &mut arr)
.map_err(|e| SdkError::ClientError(format!("bad hash from server: {e}")))?;
missing.insert(arr);
}
}
Ok(missing)
}
pub async fn commit_job_status(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
job_id: &str,
) -> Result<Traced<CommitJobStatus>, SdkError> {
self.commit_job_get(project_id, repo, git_username, git_token, job_id, None)
.await
}
async fn commit_job_get(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
job_id: &str,
wait_generation: Option<u64>,
) -> Result<Traced<CommitJobStatus>, SdkError> {
let id = super::encode_path_segment(job_id);
let (suffix, timeout) = match wait_generation {
Some(g) => (
format!(
"commits/jobs/{id}?wait_generation={g}&timeout_ms={JOB_LONG_POLL_TIMEOUT_MS}"
),
std::time::Duration::from_millis(JOB_LONG_POLL_TIMEOUT_MS + 10_000),
),
None => (
format!("commits/jobs/{id}"),
std::time::Duration::from_secs(30),
),
};
let (req, trace_id) = self.git_request(
Method::GET,
project_id,
repo,
Some(&suffix),
git_username,
git_token,
)?;
let value = expect_json(req.timeout(timeout).send().await?).await?;
Ok(Traced::new(trace_id, value))
}
async fn ingest_staging_target(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
session_id: &str,
) -> Result<IngestStagingWire, SdkError> {
with_transient_retries(|| async {
let (req, _t) = self.git_request(
Method::POST,
project_id,
repo,
Some(&format!("ingest/sessions/{session_id}/staging")),
git_username,
git_token,
)?;
expect_json(req.send().await?).await
})
.await
}
#[allow(clippy::too_many_arguments)]
async fn upload_staged_pack(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
session_id: &str,
target: IngestStagingWire,
pack: Vec<u8>,
entries: Vec<StagedEntryWire>,
) -> Result<(), SdkError> {
let url = target.url.as_deref().ok_or_else(|| {
SdkError::ClientError("staged upload requires a presigned url".to_string())
})?;
let pack = bytes::Bytes::from(pack);
with_transient_retries(|| {
let pack = pack.clone();
async move {
let resp = self.git_client.put(url).body(pack).send().await?;
if !resp.status().is_success() {
return Err(SdkError::ServerError {
status: resp.status(),
message: format!(
"staged pack PUT: {}",
resp.text().await.unwrap_or_default()
),
});
}
Ok(())
}
})
.await?;
const REGISTER_BATCH: usize = 4096;
let mut remaining = entries;
while !remaining.is_empty() {
let take = remaining.len().min(REGISTER_BATCH);
let batch: Vec<StagedEntryWire> = remaining.drain(..take).collect();
let body = StagedRegisterWire {
pack_id: target.pack_id.clone(),
entries: batch,
};
with_transient_retries(|| async {
let (req, _t) = self.git_request(
Method::POST,
project_id,
repo,
Some(&format!("ingest/sessions/{session_id}/staged")),
git_username,
git_token,
)?;
expect_ok(req.json(&body).send().await?).await
})
.await?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
async fn put_batch_files(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
session_id: &str,
body: Vec<u8>,
) -> Result<BatchUploadRespWire, SdkError> {
let body = bytes::Bytes::from(body);
with_transient_retries(|| {
let body = body.clone();
async move {
let (req, _t) = self.git_request(
Method::PUT,
project_id,
repo,
Some(&format!("ingest/sessions/{session_id}/files")),
git_username,
git_token,
)?;
expect_json(req.body(body).send().await?).await
}
})
.await
}
#[allow(clippy::too_many_arguments)]
async fn put_chunk_frame(
&self,
project_id: &str,
repo: &str,
git_username: &str,
git_token: &str,
session_id: &str,
file: Option<(&str, u64, u64)>,
frame: Vec<u8>,
) -> Result<(), SdkError> {
let suffix = match file {
None => format!("ingest/sessions/{session_id}/chunks"),
Some((token, total, offset)) => format!(
"ingest/sessions/{session_id}/chunks?file={token}&file_total={total}&file_offset={offset}"
),
};
let retriable = file.is_none();
let frame = bytes::Bytes::from(frame);
let send = || {
let frame = frame.clone();
let suffix = suffix.clone();
async move {
let (req, _t) = self.git_request(
Method::PUT,
project_id,
repo,
Some(&suffix),
git_username,
git_token,
)?;
expect_ok(req.body(frame).send().await?).await
}
};
if retriable {
with_transient_retries(send).await
} else {
send().await
}
}
}
fn frame_payload_bytes(frame: &[u8]) -> u64 {
let mut cursor = frame;
let mut total = 0u64;
while cursor.len() >= 36 {
let len = u32::from_be_bytes(cursor[32..36].try_into().expect("sized")) as u64;
total += len;
cursor = &cursor[36 + len as usize..];
}
total
}
fn hex_lower(h: &[u8; 32]) -> String {
hex::encode(h)
}
pub(super) async fn expect_json<T: serde::de::DeserializeOwned>(
resp: reqwest::Response,
) -> Result<T, SdkError> {
let status = resp.status();
if !status.is_success() {
let message = resp.text().await.unwrap_or_default();
return Err(SdkError::ServerError { status, message });
}
resp.json::<T>()
.await
.map_err(|e| SdkError::ClientError(format!("bad response body: {e}")))
}
async fn expect_ok(resp: reqwest::Response) -> Result<(), SdkError> {
let status = resp.status();
if !status.is_success() {
let message = resp.text().await.unwrap_or_default();
return Err(SdkError::ServerError { status, message });
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn checkpoint_file_paths_encode_components_without_hiding_separators() {
assert_eq!(
encoded_repo_path("src/what ?#%.rs", "test path").unwrap(),
"src/what%20%3F%23%25.rs"
);
assert_eq!(
encoded_repo_path("日本語/é.rs", "test path").unwrap(),
"%E6%97%A5%E6%9C%AC%E8%AA%9E/%C3%A9.rs"
);
assert!(encoded_repo_path("../secret", "test path").is_err());
}
#[test]
fn unknown_publish_outcomes_fail_closed() {
let response: MaterializeWorkspaceCheckpointResponse =
serde_json::from_value(serde_json::json!({
"generation": 1,
"commit": "a".repeat(40),
"tree": "b".repeat(40),
"ref_name": "refs/workspaces/w",
"parent": null,
"created": true,
"publish_status": "new_server_outcome",
"published_commit": null,
"publish_error": null,
}))
.unwrap();
assert_eq!(
response.publish_status,
WorkspaceCheckpointPublishStatus::Unknown
);
assert!(!response.publish_status.is_success());
}
fn cf(chunks: Vec<([u8; 32], u32)>, delete: bool, known_oid: bool) -> ChunkedFile {
let cdc = !delete && !known_oid && !chunks.is_empty();
ChunkedFile {
repo_path: "p".to_string(),
source: PushSource::Bytes(Vec::new()),
chunks,
mode: None,
delete,
known_oid,
blob_oid: String::new(),
cdc,
}
}
fn h(b: u8) -> [u8; 32] {
[b; 32]
}
#[test]
fn subtree_prefix_is_applied_to_every_change_kind() {
let mut files = vec![
PushFile {
repo_path: "src/lib.rs".to_string(),
source: PushSource::Bytes(b"new".to_vec()),
mode: Some(0o100644),
delete: false,
},
PushFile {
repo_path: "old.rs".to_string(),
source: PushSource::Bytes(Vec::new()),
mode: None,
delete: true,
},
PushFile {
repo_path: "renamed.rs".to_string(),
source: PushSource::KnownOid("1".repeat(40)),
mode: Some(0o100644),
delete: false,
},
];
apply_path_prefix(&mut files, Some("services/auth")).unwrap();
assert_eq!(files[0].repo_path, "services/auth/src/lib.rs");
assert_eq!(files[1].repo_path, "services/auth/old.rs");
assert_eq!(files[2].repo_path, "services/auth/renamed.rs");
}
#[test]
fn subtree_prefix_rejects_paths_that_can_escape_or_alias() {
for bad in ["", "/root", "root/", "a//b", "a/./b", "a/../b"] {
let mut files = vec![PushFile {
repo_path: "file".to_string(),
source: PushSource::Bytes(Vec::new()),
mode: None,
delete: false,
}];
assert!(apply_path_prefix(&mut files, Some(bad)).is_err(), "{bad:?}");
}
let mut files = vec![PushFile {
repo_path: "../outside".to_string(),
source: PushSource::Bytes(Vec::new()),
mode: None,
delete: true,
}];
assert!(apply_path_prefix(&mut files, Some("services/auth")).is_err());
}
#[test]
fn workspace_checkpoint_wire_reuses_commit_file_declarations_without_commit_fields() {
let files = vec![CommitFileWire {
path: "src/lib.rs".to_string(),
chunks: vec![CommitChunkWire {
hash: "ab".repeat(32),
size: 17,
}],
content: None,
file_token: None,
oid: None,
delete: false,
mode: Some("100644".to_string()),
}];
let base = "1".repeat(40);
let wire = WorkspaceCheckpointWire {
format_ver: 1,
generation: 7,
checkpoint_id: "checkpoint-7",
base: &base,
base_ref: Some("refs/heads/main"),
publish_target: Some("main"),
mount_id: Some("mount-1"),
mounted_on: Some("sandbox:/code"),
session_id: "session-1",
files: &files,
};
let value = serde_json::to_value(wire).unwrap();
assert_eq!(value["format_ver"], 1);
assert_eq!(value["generation"], 7);
assert_eq!(value["checkpoint_id"], "checkpoint-7");
assert_eq!(value["base"], "1".repeat(40));
assert_eq!(value["base_ref"], "refs/heads/main");
assert_eq!(value["publish_target"], "main");
assert_eq!(value["mount_id"], "mount-1");
assert_eq!(value["mounted_on"], "sandbox:/code");
assert_eq!(value["session_id"], "session-1");
assert_eq!(value["files"][0]["path"], "src/lib.rs");
assert_eq!(value["files"][0]["chunks"][0]["size"], 17);
assert!(value.get("branch").is_none());
assert!(value.get("message").is_none());
assert!(value.get("expect_oid").is_none());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn workspace_checkpoint_endpoints_retry_and_preserve_wire() {
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
#[derive(Clone, Debug)]
struct RecordedRequest {
method: String,
target: String,
body: Vec<u8>,
range: Option<String>,
}
let workspace_id = "a".repeat(40);
let base_oid = "b".repeat(40);
let snapshot_tip = "c".repeat(40);
let checkpoint_tree = "d".repeat(40);
let materialized_commit = "e".repeat(40);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let seen: Arc<Mutex<Vec<RecordedRequest>>> = Arc::default();
let attempts: Arc<Mutex<HashMap<String, usize>>> = Arc::default();
let seen_server = seen.clone();
let attempts_server = attempts.clone();
let checkpoint_path = format!("/project/p/repos/r/workspaces/{workspace_id}/checkpoint");
let changes_path = format!("{checkpoint_path}/changes");
let checkpoint_file_path = format!("{checkpoint_path}/files/src/main.rs");
let materialize_path = format!("{checkpoint_path}/materialize");
let checkpoint_path_server = checkpoint_path.clone();
let changes_path_server = changes_path.clone();
let checkpoint_file_path_server = checkpoint_file_path.clone();
let materialize_path_server = materialize_path.clone();
let server = tokio::spawn({
let workspace_id = workspace_id.clone();
let base_oid = base_oid.clone();
let snapshot_tip = snapshot_tip.clone();
let checkpoint_tree = checkpoint_tree.clone();
let materialized_commit = materialized_commit.clone();
async move {
loop {
let Ok((mut socket, _)) = listener.accept().await else {
return;
};
let seen = seen_server.clone();
let attempts = attempts_server.clone();
let workspace_id = workspace_id.clone();
let base_oid = base_oid.clone();
let snapshot_tip = snapshot_tip.clone();
let checkpoint_tree = checkpoint_tree.clone();
let materialized_commit = materialized_commit.clone();
let checkpoint_path = checkpoint_path_server.clone();
let changes_path = changes_path_server.clone();
let checkpoint_file_path = checkpoint_file_path_server.clone();
let materialize_path = materialize_path_server.clone();
tokio::spawn(async move {
let mut buffered = Vec::new();
loop {
let head_end = loop {
if let Some(pos) =
buffered.windows(4).position(|window| window == b"\r\n\r\n")
{
break pos + 4;
}
let mut chunk = [0_u8; 4096];
match socket.read(&mut chunk).await {
Ok(0) | Err(_) => return,
Ok(read) => buffered.extend_from_slice(&chunk[..read]),
}
};
let head = String::from_utf8_lossy(&buffered[..head_end]).to_string();
let request_line = head.lines().next().unwrap_or_default();
let mut request_parts = request_line.split_whitespace();
let method = request_parts.next().unwrap_or_default().to_string();
let target = request_parts.next().unwrap_or_default().to_string();
let content_length = head
.lines()
.find_map(|line| {
line.to_ascii_lowercase()
.strip_prefix("content-length:")
.and_then(|value| value.trim().parse::<usize>().ok())
})
.unwrap_or(0);
let range = head.lines().find_map(|line| {
let (name, value) = line.split_once(':')?;
name.eq_ignore_ascii_case("range")
.then(|| value.trim().to_string())
});
while buffered.len() < head_end + content_length {
let mut chunk = [0_u8; 4096];
match socket.read(&mut chunk).await {
Ok(0) | Err(_) => return,
Ok(read) => buffered.extend_from_slice(&chunk[..read]),
}
}
let body = buffered[head_end..head_end + content_length].to_vec();
buffered.drain(..head_end + content_length);
seen.lock().unwrap().push(RecordedRequest {
method: method.clone(),
target: target.clone(),
body,
range: range.clone(),
});
let path = target.split('?').next().unwrap_or_default();
let route = format!("{method} {path}");
let attempt = {
let mut attempts = attempts.lock().unwrap();
let count = attempts.entry(route).or_default();
*count += 1;
*count
};
let (status, response_body) = if method == "GET"
&& path == checkpoint_file_path
&& range.as_deref() == Some("bytes=0-4")
{
("206 Partial Content", "check".to_string())
} else if path.ends_with("/ingest/sessions") {
(
"200 OK",
serde_json::json!({
"session_id": "session-1",
"cdc_min_bytes": 1024,
"cdc_avg_bytes": 4096,
"cdc_max_bytes": 16384,
"max_hashes_per_query": 512,
"max_chunk_bytes": 1_048_576,
"features": ["oid_files"]
})
.to_string(),
)
} else if method == "PUT" && path == checkpoint_path {
if attempt == 1 {
(
"503 Service Unavailable",
serde_json::json!({"message": "retry checkpoint"})
.to_string(),
)
} else {
(
"201 Created",
serde_json::json!({
"format_ver": 1,
"workspace_id": workspace_id,
"ref_name": format!("refs/workspaces/{workspace_id}"),
"generation": 7,
"checkpoint_id": "checkpoint-7",
"base": base_oid,
"snapshot_tip": snapshot_tip,
"tree": checkpoint_tree,
"created_at_ms": 10,
"updated_at_ms": 11,
"materialized_generation": null
})
.to_string(),
)
}
} else if method == "GET" && path == checkpoint_path {
(
"200 OK",
serde_json::json!({
"format_ver": 1,
"workspace_id": workspace_id,
"ref_name": format!("refs/workspaces/{workspace_id}"),
"generation": 7,
"checkpoint_id": "checkpoint-7",
"base": base_oid,
"snapshot_tip": snapshot_tip,
"tree": checkpoint_tree,
"created_at_ms": 10,
"updated_at_ms": 11,
"materialized_generation": null
})
.to_string(),
)
} else if method == "GET" && path == changes_path {
(
"200 OK",
serde_json::json!({
"generation": 7,
"checkpoint_id": "checkpoint-7",
"snapshot_tip": snapshot_tip,
"tree": checkpoint_tree,
"materialized_generation": null,
"entries": [{
"path": "src/main.rs",
"change": "modified",
"mode": 0o100644,
"size": 12,
"oid": "f".repeat(40)
}],
"truncated": false,
"next_after": null
})
.to_string(),
)
} else if method == "GET" && path == checkpoint_file_path {
("200 OK", "checkpoint-body".to_string())
} else if method == "POST" && path == materialize_path {
if attempt == 1 {
(
"503 Service Unavailable",
serde_json::json!({"message": "retry materialize"})
.to_string(),
)
} else {
(
"201 Created",
serde_json::json!({
"generation": 7,
"commit": materialized_commit,
"tree": checkpoint_tree,
"ref_name": format!("refs/workspaces/{workspace_id}"),
"parent": snapshot_tip,
"created": true,
"publish_status": "published",
"published_commit": "9".repeat(40),
"publish_error": null
})
.to_string(),
)
}
} else {
(
"404 Not Found",
serde_json::json!({
"message": format!("unhandled test route {method} {target}")
})
.to_string(),
)
};
let range_headers = if status == "206 Partial Content" {
"content-range: bytes 0-4/15\r\naccept-ranges: bytes\r\n"
} else {
""
};
let response = format!(
"HTTP/1.1 {status}\r\ncontent-type: application/json\r\n{range_headers}content-length: {}\r\n\r\n{response_body}",
response_body.len()
);
if socket.write_all(response.as_bytes()).await.is_err() {
return;
}
}
});
}
}
});
let base_url = format!("http://{addr}");
let client = crate::ClientBuilder::new(&base_url)
.bearer_token("dummy")
.build()
.unwrap();
let sdk = ArtifactStorageClient::new(client, &base_url).unwrap();
let checkpoint = sdk
.push_workspace_checkpoint(
"p",
"r",
"u",
"tok",
vec![PushFile {
repo_path: "src/main.rs".to_string(),
source: PushSource::KnownOid("f".repeat(40)),
mode: Some(0o100644),
delete: false,
}],
PushOptions {
base: Some(base_oid.clone()),
..Default::default()
},
WorkspaceCheckpointOptions {
workspace_id: workspace_id.clone(),
generation: 7,
checkpoint_id: "checkpoint-7".to_string(),
base_ref: Some("refs/heads/main".to_string()),
publish_target: Some("main".to_string()),
mount_id: Some("mount-1".to_string()),
mounted_on: Some("sandbox:/code".to_string()),
},
)
.await
.expect("checkpoint PUT must retry a 503")
.into_inner();
assert_eq!(checkpoint.checkpoint.tree, checkpoint_tree);
let read = sdk
.workspace_checkpoint("p", "r", "u", "tok", &workspace_id)
.await
.unwrap()
.into_inner();
assert_eq!(read.generation, 7);
let changes = sdk
.workspace_checkpoint_changes(
"p",
"r",
"u",
"tok",
&workspace_id,
Some("src/lib.rs"),
23,
)
.await
.unwrap()
.into_inner();
assert_eq!(changes.entries[0].path, "src/main.rs");
assert_eq!(changes.checkpoint_id, "checkpoint-7");
let checkpoint_bytes = sdk
.workspace_checkpoint_file_bytes(
"p",
"r",
"u",
"tok",
&workspace_id,
"src/main.rs",
7,
"checkpoint-7",
&checkpoint_tree,
)
.await
.unwrap()
.into_inner();
assert_eq!(checkpoint_bytes, b"checkpoint-body");
let checkpoint_range = sdk
.workspace_checkpoint_file_range(
"p",
"r",
"u",
"tok",
&workspace_id,
"src/main.rs",
7,
"checkpoint-7",
&checkpoint_tree,
0,
4,
)
.await
.unwrap()
.into_inner();
assert_eq!(checkpoint_range, b"check");
let mut materialize = MaterializeWorkspaceCheckpointRequest::new(7, "intentional snapshot");
materialize.author = Some(super::super::merge::Signature {
name: "Agent".to_string(),
email: "agent@example.com".to_string(),
});
let materialized = sdk
.materialize_workspace_checkpoint("p", "r", "u", "tok", &workspace_id, &materialize)
.await
.expect("materialize POST must retry a 503")
.into_inner();
assert_eq!(materialized.commit, materialized_commit);
assert_eq!(
materialized.publish_status,
WorkspaceCheckpointPublishStatus::Published
);
assert_eq!(
materialized.published_commit.as_deref(),
Some("9999999999999999999999999999999999999999")
);
assert_eq!(materialized.publish_error, None);
server.abort();
let seen = seen.lock().unwrap().clone();
let checkpoint_puts: Vec<_> = seen
.iter()
.filter(|request| request.method == "PUT" && request.target == checkpoint_path)
.collect();
assert_eq!(checkpoint_puts.len(), 2, "503 checkpoint PUT must retry");
assert_eq!(checkpoint_puts[0].body, checkpoint_puts[1].body);
let checkpoint_body: serde_json::Value =
serde_json::from_slice(&checkpoint_puts[0].body).unwrap();
assert_eq!(checkpoint_body["format_ver"], 1);
assert_eq!(checkpoint_body["generation"], 7);
assert_eq!(checkpoint_body["checkpoint_id"], "checkpoint-7");
assert_eq!(checkpoint_body["base"], base_oid);
assert_eq!(checkpoint_body["base_ref"], "refs/heads/main");
assert_eq!(checkpoint_body["publish_target"], "main");
assert_eq!(checkpoint_body["mount_id"], "mount-1");
assert_eq!(checkpoint_body["mounted_on"], "sandbox:/code");
assert_eq!(checkpoint_body["session_id"], "session-1");
assert_eq!(checkpoint_body["files"][0]["path"], "src/main.rs");
assert_eq!(checkpoint_body["files"][0]["oid"], "f".repeat(40));
assert_eq!(checkpoint_body["files"][0]["mode"], "100644");
assert!(checkpoint_body.get("branch").is_none());
assert!(checkpoint_body.get("message").is_none());
assert_eq!(
seen.iter()
.filter(|request| request.method == "GET" && request.target == checkpoint_path)
.count(),
1
);
let changes_request = seen
.iter()
.find(|request| request.method == "GET" && request.target.starts_with(&changes_path))
.expect("changes GET");
let changes_url =
reqwest::Url::parse(&format!("http://test{}", changes_request.target)).unwrap();
let query: HashMap<_, _> = changes_url.query_pairs().into_owned().collect();
assert_eq!(query.get("limit").map(String::as_str), Some("23"));
assert_eq!(query.get("after").map(String::as_str), Some("src/lib.rs"));
let checkpoint_file_request = seen
.iter()
.find(|request| {
request.method == "GET" && request.target.starts_with(&checkpoint_file_path)
})
.expect("revision-bound checkpoint file GET");
let checkpoint_file_url =
reqwest::Url::parse(&format!("http://test{}", checkpoint_file_request.target)).unwrap();
let file_query: HashMap<_, _> = checkpoint_file_url.query_pairs().into_owned().collect();
assert_eq!(file_query.get("generation").map(String::as_str), Some("7"));
assert_eq!(
file_query.get("checkpoint_id").map(String::as_str),
Some("checkpoint-7")
);
assert_eq!(
file_query.get("tree").map(String::as_str),
Some(checkpoint_tree.as_str())
);
assert!(seen.iter().any(|request| {
request.method == "GET"
&& request.target.starts_with(&checkpoint_file_path)
&& request.range.as_deref() == Some("bytes=0-4")
}));
let materialize_posts: Vec<_> = seen
.iter()
.filter(|request| request.method == "POST" && request.target == materialize_path)
.collect();
assert_eq!(
materialize_posts.len(),
2,
"503 materialize POST must retry"
);
assert_eq!(materialize_posts[0].body, materialize_posts[1].body);
let materialize_body: serde_json::Value =
serde_json::from_slice(&materialize_posts[0].body).unwrap();
assert_eq!(materialize_body["format_ver"], 1);
assert_eq!(materialize_body["generation"], 7);
assert_eq!(materialize_body["message"], "intentional snapshot");
assert_eq!(materialize_body["publish_configured"], true);
assert_eq!(materialize_body["author"]["name"], "Agent");
assert!(materialize_body.get("committer").is_none());
}
#[tokio::test(start_paused = true)]
async fn transient_retries_back_off_and_stop_on_client_errors() {
use std::sync::atomic::{AtomicUsize, Ordering};
let calls = AtomicUsize::new(0);
let out: Result<u32, SdkError> = with_transient_retries(|| {
let n = calls.fetch_add(1, Ordering::SeqCst);
async move {
if n < 2 {
Err(SdkError::ServerError {
status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
message: "wobble".into(),
})
} else {
Ok(7)
}
}
})
.await;
assert_eq!(out.unwrap(), 7);
assert_eq!(
calls.load(Ordering::SeqCst),
3,
"two transient failures retried"
);
let calls = AtomicUsize::new(0);
let out: Result<u32, SdkError> = with_transient_retries(|| {
calls.fetch_add(1, Ordering::SeqCst);
async {
Err(SdkError::ServerError {
status: reqwest::StatusCode::BAD_REQUEST,
message: "no".into(),
})
}
})
.await;
assert!(out.is_err());
assert_eq!(
calls.load(Ordering::SeqCst),
1,
"4xx must not retry: the rejection is deterministic"
);
let calls = AtomicUsize::new(0);
let out: Result<u32, SdkError> = with_transient_retries(|| {
calls.fetch_add(1, Ordering::SeqCst);
async {
Err(SdkError::ServerError {
status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
message: "down".into(),
})
}
})
.await;
assert!(out.is_err());
assert_eq!(calls.load(Ordering::SeqCst), 1 + TRANSIENT_RETRIES);
}
#[test]
fn tokened_election_smalls_always_larges_by_ratio() {
let missing: std::collections::HashSet<[u8; 32]> = [h(1), h(2)].into();
let files = vec![
cf(vec![(h(9), 100)], false, false), cf(vec![(h(1), 300), (h(9), 300)], false, false), cf(vec![(h(9), 300), (h(8), 300)], false, false), cf(vec![(h(2), 100)], true, false), cf(Vec::new(), false, true), cf(Vec::new(), false, false), ];
assert_eq!(
elect_tokened(&files, &missing, 200),
vec![true, true, false, false, false, false]
);
assert_eq!(
elect_tokened(&files, &missing, 0),
vec![false, true, false, false, false, false]
);
}
#[test]
fn dedup_coverage_ignores_tokened_uploads() {
let missing: std::collections::HashSet<[u8; 32]> = [h(1), h(2), h(3)].into();
let files = vec![
cf(vec![(h(1), 100), (h(2), 100)], false, false),
cf(vec![(h(1), 100), (h(3), 100), (h(9), 800)], false, false),
];
let tokened = vec![true, false];
let need = dedup_needed_chunks(&files, &tokened, &missing);
assert!(need.contains(&h(1)), "shared chunk must upload via dedup");
assert!(need.contains(&h(3)));
assert!(
!need.contains(&h(2)),
"chunks only a tokened file carries ride the tokened stream"
);
}
#[test]
fn worktree_repo_paths_from_dot_have_no_dot_components() {
let root = std::path::Path::new(".");
assert_eq!(
repo_path_for_worktree(root, std::path::Path::new("./.clang-format")),
".clang-format"
);
assert_eq!(
repo_path_for_worktree(root, std::path::Path::new("./arch/boot.c")),
"arch/boot.c"
);
}
#[test]
fn worktree_repo_paths_from_absolute_root_are_root_relative() {
let root = std::path::Path::new("/tmp/somedir");
assert_eq!(
repo_path_for_worktree(root, std::path::Path::new("/tmp/somedir/a/b.txt")),
"a/b.txt"
);
}
#[test]
fn collect_worktree_files_skips_git_metadata() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("a.txt"), "a").unwrap();
std::fs::create_dir(dir.path().join(".git")).unwrap();
std::fs::write(dir.path().join(".git").join("config"), "ignored").unwrap();
let mut files = Vec::new();
collect_worktree_files(dir.path(), &mut files).unwrap();
assert_eq!(files.len(), 1);
assert_eq!(files[0].repo_path, "a.txt");
}
#[test]
fn collect_worktree_files_honors_gitignore_without_git_dir() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join(".gitignore"), "ignored.bin\n").unwrap();
std::fs::write(dir.path().join("ignored.bin"), "ignored").unwrap();
std::fs::write(dir.path().join("kept.tensorlake-test"), "kept").unwrap();
let mut files = Vec::new();
collect_worktree_files(dir.path(), &mut files).unwrap();
let paths: Vec<_> = files.iter().map(|file| file.repo_path.as_str()).collect();
assert_eq!(paths, vec![".gitignore", "kept.tensorlake-test"]);
}
#[cfg(unix)]
#[test]
fn collect_worktree_files_preserves_executable_mode() {
use std::os::unix::fs::PermissionsExt;
let dir = tempfile::tempdir().unwrap();
let script = dir.path().join("script.sh");
std::fs::write(&script, "#!/bin/sh\n").unwrap();
std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap();
let mut files = Vec::new();
collect_worktree_files(dir.path(), &mut files).unwrap();
assert_eq!(files.len(), 1);
assert_eq!(files[0].repo_path, "script.sh");
assert_eq!(files[0].mode, Some(0o100755));
}
#[cfg(unix)]
#[test]
fn collect_worktree_files_preserves_symlink_mode_and_target() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("target.txt"), "target").unwrap();
std::os::unix::fs::symlink("target.txt", dir.path().join("link.txt")).unwrap();
let mut files = Vec::new();
collect_worktree_files(dir.path(), &mut files).unwrap();
let link = files
.iter()
.find(|file| file.repo_path == "link.txt")
.expect("symlink should be collected");
assert_eq!(link.mode, Some(0o120000));
match &link.source {
PushSource::Bytes(bytes) => assert_eq!(bytes, b"target.txt"),
other => panic!("expected symlink bytes, got {other:?}"),
}
}
#[test]
fn known_oid_commit_wire_shape() {
let wire = CommitFileWire {
path: "copies/b.txt".to_string(),
chunks: Vec::new(),
content: None,
file_token: None,
oid: Some("00112233445566778899aabbccddeeff00112233".to_string()),
delete: false,
mode: None,
};
let json = serde_json::to_value(&wire).unwrap();
assert_eq!(json["oid"], "00112233445566778899aabbccddeeff00112233");
assert!(json.get("chunks").is_none());
assert!(json.get("content").is_none());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn detached_snapshot_push_polls_commit_job_route() {
use std::sync::{Arc, Mutex};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
fn respond(path: &str) -> (&'static str, String) {
if path.ends_with("/ingest/sessions") {
(
"200 OK",
serde_json::json!({
"session_id": "s1",
"cdc_min_bytes": 1024,
"cdc_avg_bytes": 4096,
"cdc_max_bytes": 16384,
"max_hashes_per_query": 512,
"max_chunk_bytes": 1_048_576,
"features": ["oid_files"],
})
.to_string(),
)
} else if path.contains("/workspaces/ws-1/snapshots") {
(
"202 Accepted",
serde_json::json!({
"job_id": "j-1",
"state": "running",
"generation": 1,
"poll_url": "/project/p/repos/r/commits/jobs/j-1",
})
.to_string(),
)
} else if path.contains("/commits/jobs/j-1") {
(
"200 OK",
serde_json::json!({
"job_id": "j-1",
"state": "committed",
"generation": 3,
"commit": "c".repeat(40),
"tree": "d".repeat(40),
"ref_name": "refs/workspaces/ws-1",
"created": true,
})
.to_string(),
)
} else if path.ends_with("/commits") {
(
"201 Created",
serde_json::json!({
"job_id": "j-2",
"state": "committed",
"generation": 2,
"commit": "e".repeat(40),
"tree": "d".repeat(40),
"ref_name": "refs/heads/main",
"created": true,
})
.to_string(),
)
} else {
(
"404 Not Found",
serde_json::json!({ "message": "no such route (test server)" }).to_string(),
)
}
}
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let seen: Arc<Mutex<Vec<String>>> = Arc::default();
let seen_srv = seen.clone();
let server = tokio::spawn(async move {
loop {
let Ok((mut sock, _)) = listener.accept().await else {
return;
};
let seen = seen_srv.clone();
tokio::spawn(async move {
let mut buf: Vec<u8> = Vec::new();
loop {
let head_end = loop {
if let Some(pos) = buf.windows(4).position(|w| w == b"\r\n\r\n") {
break pos + 4;
}
let mut tmp = [0u8; 4096];
match sock.read(&mut tmp).await {
Ok(0) | Err(_) => return,
Ok(n) => buf.extend_from_slice(&tmp[..n]),
}
};
let head = String::from_utf8_lossy(&buf[..head_end]).to_string();
let request_line = head.lines().next().unwrap_or_default().to_string();
let content_length: usize = head
.lines()
.find_map(|l| {
l.to_ascii_lowercase()
.strip_prefix("content-length:")
.map(|v| v.trim().parse().unwrap_or(0))
})
.unwrap_or(0);
while buf.len() < head_end + content_length {
let mut tmp = [0u8; 4096];
match sock.read(&mut tmp).await {
Ok(0) | Err(_) => return,
Ok(n) => buf.extend_from_slice(&tmp[..n]),
}
}
buf.drain(..head_end + content_length);
seen.lock().unwrap().push(request_line.clone());
let path = request_line
.split_whitespace()
.nth(1)
.unwrap_or_default()
.to_string();
let (status, body) = respond(&path);
let resp = format!(
"HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\n\r\n{body}",
body.len()
);
if sock.write_all(resp.as_bytes()).await.is_err() {
return;
}
}
});
}
});
let base = format!("http://{addr}");
let client = crate::ClientBuilder::new(&base)
.bearer_token("dummy")
.build()
.unwrap();
let sdk = ArtifactStorageClient::new(client, &base).unwrap();
let report = sdk
.push_files(
"p",
"r",
"u",
"tok",
vec![PushFile {
repo_path: "a.txt".to_string(),
source: PushSource::KnownOid("a".repeat(40)),
mode: None,
delete: false,
}],
PushOptions {
message: "m".to_string(),
workspace_snapshot: Some("ws-1".to_string()),
..Default::default()
},
)
.await
.expect("push must survive a 202 detach")
.into_inner();
let branch_report = sdk
.push_files(
"p",
"r",
"u",
"tok",
vec![PushFile {
repo_path: "b.txt".to_string(),
source: PushSource::KnownOid("b".repeat(40)),
mode: None,
delete: false,
}],
PushOptions {
message: "m2".to_string(),
branch: "main".to_string(),
..Default::default()
},
)
.await
.expect("branch push must succeed")
.into_inner();
server.abort();
assert_eq!(report.commit, "c".repeat(40));
assert_eq!(branch_report.commit, "e".repeat(40));
let seen = seen.lock().unwrap().clone();
assert!(
seen.iter()
.any(|l| l.starts_with("POST /project/p/repos/r/workspaces/ws-1/snapshots")),
"snapshot submit must use the workspace route; saw: {seen:?}"
);
assert!(
seen.iter()
.any(|l| l.starts_with("POST /project/p/repos/r/commits HTTP")),
"branch submit must use the commits route; saw: {seen:?}"
);
let polls: Vec<&String> = seen.iter().filter(|l| l.starts_with("GET ")).collect();
assert!(
!polls.is_empty(),
"a 202 detach must be followed by at least one poll; saw: {seen:?}"
);
for poll in polls {
assert!(
poll.starts_with("GET /project/p/repos/r/commits/jobs/j-1"),
"detached job polled the wrong route: {poll}"
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "requires a local artifact-storage server on 127.0.0.1:8080"]
async fn push_paths_roundtrip_against_local_server() {
const BASE: &str = "http://127.0.0.1:8080";
if std::net::TcpStream::connect_timeout(
&"127.0.0.1:8080".parse().unwrap(),
std::time::Duration::from_millis(500),
)
.is_err()
{
eprintln!("skipping: no local artifact-storage server");
return;
}
let client = crate::ClientBuilder::new(BASE)
.bearer_token("dummy")
.build()
.unwrap();
let sdk = ArtifactStorageClient::new(client, BASE).unwrap();
let repo = format!(
"ingest-{}",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
);
sdk.create_repo_with_credential("ingesttest", &repo, None, None, "t", "devtoken")
.await
.unwrap();
let salt = repo.as_bytes().to_vec();
let content_a: Vec<u8> = [b"alpha ".as_slice(), &salt].concat();
let big: Vec<u8> = (0..3usize * 1024 * 1024)
.map(|i| {
(i as u8)
.wrapping_mul(31)
.wrapping_add(salt[i % salt.len()])
})
.collect();
let report = sdk
.push_files(
"ingesttest",
&repo,
"t",
"devtoken",
vec![
PushFile {
repo_path: "a.txt".to_string(),
source: PushSource::Bytes(content_a.clone()),
mode: None,
delete: false,
},
PushFile {
repo_path: "dup/a-again.txt".to_string(),
source: PushSource::Bytes(content_a.clone()),
mode: None,
delete: false,
},
PushFile {
repo_path: "big.bin".to_string(),
source: PushSource::Bytes(big),
mode: None,
delete: false,
},
PushFile {
repo_path: "empty.txt".to_string(),
source: PushSource::Bytes(Vec::new()),
mode: None,
delete: false,
},
],
PushOptions {
message: "seed".into(),
..Default::default()
},
)
.await
.unwrap()
.into_inner();
assert_eq!(report.files, 4);
assert_eq!(
report
.file_blob_oids
.iter()
.find(|(p, _)| p == "empty.txt")
.unwrap()
.1,
"e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
"empty file must carry git's empty-blob oid"
);
let a_oid = report
.file_blob_oids
.iter()
.find(|(p, _)| p == "a.txt")
.unwrap()
.1
.clone();
let dup_oid = &report
.file_blob_oids
.iter()
.find(|(p, _)| p == "dup/a-again.txt")
.unwrap()
.1;
assert_eq!(&a_oid, dup_oid, "identical content has one blob oid");
let report2 = sdk
.push_files(
"ingesttest",
&repo,
"t",
"devtoken",
vec![PushFile {
repo_path: "copies/by-ref.txt".to_string(),
source: PushSource::KnownOid(a_oid.clone()),
mode: None,
delete: false,
}],
PushOptions {
message: "by reference".into(),
..Default::default()
},
)
.await
.unwrap()
.into_inner();
assert_eq!(report2.bytes_uploaded, 0, "oid references upload nothing");
assert_eq!(
report2.file_blob_oids,
vec![("copies/by-ref.txt".to_string(), a_oid)]
);
let bogus = sdk
.push_files(
"ingesttest",
&repo,
"t",
"devtoken",
vec![PushFile {
repo_path: "copies/bogus.txt".to_string(),
source: PushSource::KnownOid(
"00112233445566778899aabbccddeeff00112233".to_string(),
),
mode: None,
delete: false,
}],
PushOptions {
message: "bogus".into(),
..Default::default()
},
)
.await;
assert!(bogus.is_err(), "unknown oid must fail the commit");
}
#[test]
fn chunking_is_deterministic_and_frames_are_well_formed() {
let data: Vec<u8> = (0..3_000_000usize).map(|i| (i % 251) as u8).collect();
let src = PushSource::Bytes(data.clone());
let (a, oid_a) = chunk_source(&src, 256 * 1024, 1024 * 1024, 4 * 1024 * 1024).unwrap();
let (b, oid_b) = chunk_source(&src, 256 * 1024, 1024 * 1024, 4 * 1024 * 1024).unwrap();
assert_eq!(a, b, "CDC must be deterministic across passes");
assert_eq!(oid_a, oid_b, "blob oid must be deterministic");
let mut reference = BlobOidHasher::new(data.len() as u64);
reference.update(&data);
assert_eq!(oid_a, reference.finalize_hex());
assert_eq!(
a.iter().map(|(_, s)| *s as usize).sum::<usize>(),
data.len(),
"chunks must cover the file exactly"
);
let (hash, size) = a[0];
let mut frame = Vec::new();
frame.extend_from_slice(&hash);
frame.extend_from_slice(&size.to_be_bytes());
frame.extend_from_slice(&data[..size as usize]);
assert_eq!(&frame[..32], &hash);
assert_eq!(u32::from_be_bytes(frame[32..36].try_into().unwrap()), size);
assert_eq!(
<[u8; 32]>::from(Sha256::digest(&frame[36..])),
hash,
"framed bytes must hash to the declared id"
);
}
#[test]
fn whole_file_pass_yields_no_chunks_for_empty_files() {
let (chunks, oid) = chunk_source_whole(&PushSource::Bytes(Vec::new())).unwrap();
assert!(chunks.is_empty(), "an empty file must have no chunks");
assert_eq!(
oid, "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
"must still hash to git's empty-blob oid"
);
let (cdc_chunks, cdc_oid) =
chunk_source(&PushSource::Bytes(Vec::new()), 256, 1024, 4096).unwrap();
assert_eq!(chunks, cdc_chunks, "both passes must agree on empty input");
assert_eq!(oid, cdc_oid);
let data = b"hello".to_vec();
let (chunks, _) = chunk_source_whole(&PushSource::Bytes(data.clone())).unwrap();
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].1 as usize, data.len());
assert_eq!(chunks[0].0, <[u8; 32]>::from(Sha256::digest(&data)));
}
#[test]
fn stable_prefix_chunking_matches_a_full_rechunk() {
let (min, avg, max) = (256 * 1024, 1024 * 1024, 4 * 1024 * 1024);
let old: Vec<u8> = (0..5_000_000usize).map(|i| (i % 251) as u8).collect();
let appended: Vec<u8> = (0..2_000_000usize).map(|i| (i % 13) as u8).collect();
let mut grown = old.clone();
grown.extend_from_slice(&appended);
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("grown.bin");
std::fs::write(&path, &grown).unwrap();
let (old_chunks, _) = chunk_source(&PushSource::Bytes(old), min, avg, max).unwrap();
let stable = &old_chunks[..old_chunks.len() - 1];
let resumed = chunk_stable_prefix(&path, stable, min, avg, max).unwrap();
let (full, _) = chunk_source(&PushSource::Path(path.clone()), min, avg, max).unwrap();
assert_eq!(
resumed, full,
"prefix + resumed tail must equal a full re-chunk exactly"
);
assert_eq!(
resumed.iter().map(|(_, s)| *s as u64).sum::<u64>(),
grown.len() as u64,
"chunks must cover the grown file exactly"
);
}
}