use clap::Args;
use fs4::{FileExt, TryLockError};
use graphforge_api::telemetry::{
ComponentHandoff, ComponentKind, ComponentRole, Failure, HandoffKind, JobFamily, JobSnapshot,
JobStage, OtlpConfig, Outcome, Stage, TelemetryConfig, TelemetryMode, TelemetryRuntime,
WaitReason,
};
use graphforge_api::{
DiscoveryPortableError, DiscoveryPortableMismatch, DiscoveryPortableRequest,
DiscoveryResearchVersionError, DiscoveryResearchVersionRequest, PortableErrorCode,
verify_discovered_portable, verify_discovered_research_version,
};
use graphforge_api::{
GraphForge, OperationId, PortableImportFootprint, PortableImportRequest, PortableLimits,
PortableMode,
};
use graphforge_discovery::{
DiscoveryError, DiscoveryErrorCode, DiscoveryLimits, DiscoveryManifest, ObjectDescriptor,
RefSet, RepositoryIdentity, ResearchLineage,
};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::fmt::Debug;
use std::fmt::Write as _;
use std::fs::{File, OpenOptions};
use std::io::{Read, Write};
#[cfg(unix)]
use std::os::unix::fs::OpenOptionsExt as _;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
use url::Url;
use crate::hub_http::{
DiscoveryCredentials, HttpResponse, HttpTransport, MAX_METADATA_BYTES, RETRY_POLICY,
RetryPolicy, RunningDigest, Transport, endpoint, fetch_discovery_with_attempts, fetch_once,
fetch_with_attempts, hash_reader, limit_error, network, parse_input_at, read_bounded, storage,
validate_url, validation,
};
#[cfg(test)]
use crate::hub_http::{fetch, parse_input};
mod module_fetch;
pub(crate) mod remote_read;
pub(crate) mod repository_listing;
pub(crate) use module_fetch::{ModuleFetchArgs, run_module_fetch};
pub(crate) use repository_listing::{HubArgs, run_hub};
pub(crate) const MAX_OBJECT_BYTES: u64 = 1024 * 1024_u64.pow(4);
const CLONE_CONTRACT: &str = "graphforge-hub-clone/1";
#[derive(Args)]
pub(crate) struct CloneArgs {
pub repository: String,
#[arg(long)]
pub hub: Option<String>,
#[arg(long)]
pub token_env: Option<String>,
#[arg(long)]
pub protection_bypass_env: Option<String>,
pub destination: Option<PathBuf>,
#[arg(long)]
pub telemetry_endpoint: Option<String>,
#[arg(long = "ref")]
pub git_ref: Option<String>,
#[arg(long)]
pub version_uuid: Option<String>,
#[arg(long)]
pub tag: Option<String>,
}
#[derive(Clone, Default)]
enum CloneClock {
#[default]
Monotonic,
#[cfg(test)]
Manual(std::rc::Rc<std::cell::Cell<Instant>>),
}
impl CloneClock {
fn now(&self) -> Instant {
match self {
Self::Monotonic => Instant::now(),
#[cfg(test)]
Self::Manual(clock) => clock.get(),
}
}
fn elapsed_since(&self, started: Instant) -> Duration {
match self {
Self::Monotonic => started.elapsed(),
#[cfg(test)]
Self::Manual(clock) => clock.get().duration_since(started),
}
}
fn wait(&self, duration: Duration) {
match self {
Self::Monotonic => std::thread::sleep(duration),
#[cfg(test)]
Self::Manual(clock) => clock.set(clock.get() + duration),
}
}
#[cfg(test)]
fn manual() -> Self {
Self::Manual(std::rc::Rc::new(std::cell::Cell::new(Instant::now())))
}
}
struct CloneProfile<'a> {
runtime: &'a TelemetryRuntime,
clock: CloneClock,
started: Option<Instant>,
cursor_ns: u64,
stages: Vec<JobStage>,
handoffs: Vec<ComponentHandoff>,
}
impl<'a> CloneProfile<'a> {
fn new(runtime: &'a TelemetryRuntime) -> Self {
Self {
runtime,
clock: CloneClock::default(),
started: None,
cursor_ns: 0,
stages: Vec::new(),
handoffs: Vec::new(),
}
}
fn handoff(
&mut self,
from: ComponentKind,
to: ComponentKind,
kind: HandoffKind,
bytes: Option<u64>,
) {
self.handoffs.push(ComponentHandoff {
start_offset_ns: self.cursor_ns,
from,
to,
kind,
duration_ns: 0,
wait_duration_ns: 0,
bytes,
records: None,
});
}
fn stage<T>(
&mut self,
stage: Stage,
component: ComponentKind,
role: ComponentRole,
wait_reason: Option<WaitReason>,
attempt: u32,
operation: impl FnOnce() -> Result<(T, Option<u64>, Option<u64>), graphforge_api::GfError>,
) -> Result<T, graphforge_api::GfError> {
let origin = *self.started.get_or_insert_with(|| self.clock.now());
let operation_start_ns =
u64::try_from(self.clock.elapsed_since(origin).as_nanos()).unwrap_or(u64::MAX);
if !self.stages.is_empty() && operation_start_ns > self.cursor_ns {
let duration_ns = operation_start_ns - self.cursor_ns;
self.stages.push(JobStage {
stage: Stage::Orchestration,
component: ComponentKind::Cli,
component_role: ComponentRole::Coordination,
start_offset_ns: self.cursor_ns,
duration_ns,
wait_duration_ns: 0,
wait_reason: None,
attempt: 1,
bytes: None,
resumed_bytes: None,
records: None,
outcome: Outcome::Ok,
});
self.cursor_ns = operation_start_ns;
}
let started = self.clock.now();
let result = operation();
let duration_ns = u64::try_from(self.clock.elapsed_since(started).as_nanos())
.unwrap_or(u64::MAX)
.max(1);
let (bytes, records) = result
.as_ref()
.map_or((None, None), |(_, bytes, records)| (*bytes, *records));
self.stages.push(JobStage {
stage,
component,
component_role: role,
start_offset_ns: self.cursor_ns,
duration_ns,
wait_duration_ns: wait_reason.map_or(0, |_| duration_ns),
wait_reason,
attempt,
bytes,
resumed_bytes: None,
records,
outcome: if result.is_ok() {
Outcome::Ok
} else {
Outcome::Failed
},
});
self.cursor_ns = self.cursor_ns.saturating_add(duration_ns);
result.map(|(value, _, _)| value)
}
fn finish(mut self, result: &Result<(), graphforge_api::GfError>) {
let elapsed_ns = self.started.map_or(0, |started| {
u64::try_from(self.clock.elapsed_since(started).as_nanos()).unwrap_or(u64::MAX)
});
if elapsed_ns > self.cursor_ns {
let duration_ns = elapsed_ns - self.cursor_ns;
self.stages.push(JobStage {
stage: Stage::Orchestration,
component: ComponentKind::Cli,
component_role: ComponentRole::Coordination,
start_offset_ns: self.cursor_ns,
duration_ns,
wait_duration_ns: 0,
wait_reason: None,
attempt: 1,
bytes: None,
resumed_bytes: None,
records: None,
outcome: if result.is_ok() {
Outcome::Ok
} else {
Outcome::Failed
},
});
self.cursor_ns = elapsed_ns;
}
let failure = result.as_ref().err().map(classify_failure);
let _ = self.runtime.record_job(JobSnapshot {
family: JobFamily::Clone,
enqueued_ns: 0,
started_ns: 0,
finished_ns: self.cursor_ns,
outcome: if result.is_ok() {
Outcome::Ok
} else {
Outcome::Failed
},
failure,
stages: self.stages,
handoffs: self.handoffs,
});
}
}
fn classify_failure(error: &graphforge_api::GfError) -> Failure {
let detail = match error {
graphforge_api::GfError::Validation(detail) | graphforge_api::GfError::Storage(detail) => {
detail.as_str()
}
_ => return Failure::Internal,
};
let code = detail.split_once(':').map_or(detail, |(code, _)| code);
match code {
"hub.network"
| "hub.unsafe_location"
| "hub.rate_limited"
| "hub.unavailable"
| "hub.unexpected_status" => Failure::Network,
"hub.limit_exceeded" | "hub.package.limit_exceeded" => Failure::ResourceLimit,
"hub.invalid_identity"
| "hub.auth_denied"
| "hub.not_found"
| "hub.access_denied"
| "hub.malformed_response"
| "hub.unsupported_future"
| "hub.missing_ref"
| "hub.missing_object"
| "hub.ambiguous_identity"
| "hub.duplicate"
| "hub.destination_conflict"
| "hub.concurrent_clone"
| "hub.interrupted"
| "hub.package.cancelled"
| "hub.package.unsupported_future"
| "hub.package.incompatible"
| "hub.integrity"
| "hub.integrity_failure"
| "hub.package.repository_mismatch"
| "hub.package.immutable_version_mismatch"
| "hub.package.package_digest_mismatch"
| "hub.package.research_version_mismatch"
| "hub.module.identity_mismatch"
| "hub.module.content_digest_mismatch"
| "hub.package.invalid_participant"
| "hub.package.invalid_structure"
| "hub.package.invalid_path"
| "hub.package.duplicate_entry"
| "hub.package.digest_mismatch"
| "hub.package.concurrent_mutation" => Failure::InvalidInput,
"hub.package.io" => Failure::Storage,
_ if matches!(error, graphforge_api::GfError::Storage(_)) => Failure::Storage,
_ => Failure::Internal,
}
}
#[derive(Serialize, Deserialize)]
struct CloneResult {
contract: String,
repository: String,
destination: String,
immutable_version: String,
package_digest: String,
generation_uuid: String,
resumed_bytes: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
resolved_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
research_branch_uuid: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
research_version_uuid: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
research_version_kind: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
research_version_identity: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
moved_from: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
moved_to: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
warnings: Vec<graphforge_api::CoreVersionWarning>,
}
struct CloneImported {
package_digest: String,
generation_uuid: uuid::Uuid,
}
struct ResearchCloneContext {
selected_branch: Option<graphforge_discovery::LineageBranch>,
lineage_bytes: Vec<u8>,
branch_uuid: String,
version_uuid: String,
version_kind: String,
identity_digest: String,
package_digest: String,
producing_core_version: String,
}
fn select_bundle(
manifest: &DiscoveryManifest,
) -> Result<&ObjectDescriptor, graphforge_api::GfError> {
manifest
.package_object()
.map_err(|error| protocol_error(&error))
}
fn fetch_inventory_object(
transport: &dyn Transport,
object: &ObjectDescriptor,
max_bytes: usize,
retry: &RetryPolicy,
) -> Result<Vec<u8>, graphforge_api::GfError> {
let location = object
.locations
.first()
.ok_or_else(|| validation("hub.missing_object", "inventory object has no location"))?;
let url = Url::parse(location)
.map_err(|_| validation("hub.unsafe_location", "inventory location is invalid"))?;
validate_url(&url)?;
let mut attempts = 0;
let response = fetch_with_attempts(
transport,
&url,
None,
None,
max_bytes as u64,
&mut attempts,
retry,
)?;
let bytes = read_bounded(response, max_bytes)?;
let actual = hash_reader(&mut std::io::Cursor::new(&bytes))?;
if actual != object.digest.0 {
return Err(validation(
"hub.integrity_failure",
"inventory object digest mismatch",
));
}
Ok(bytes)
}
fn resolve_research_version_uuid(
lineage: &ResearchLineage,
git_ref: Option<&str>,
version_uuid: Option<&str>,
) -> Result<String, graphforge_api::GfError> {
if let Some(uuid) = version_uuid {
if lineage.version(uuid).is_none() {
return Err(validation(
"hub.missing_object",
"research Version is absent from lineage",
));
}
return Ok(uuid.to_owned());
}
let git_ref = git_ref.ok_or_else(|| {
validation(
"hub.missing_ref",
"branch ref or version UUID is required for research clone",
)
})?;
let branch = lineage
.branches
.iter()
.find(|branch| branch.ref_name == git_ref)
.ok_or_else(|| {
validation(
"hub.missing_ref",
"branch ref is absent from research lineage",
)
})?;
Ok(branch.head_version_uuid.clone())
}
fn discovery_limits() -> DiscoveryLimits {
DiscoveryLimits {
max_response_bytes: MAX_METADATA_BYTES,
..DiscoveryLimits::default()
}
}
fn verify_discovery(
identity: &RepositoryIdentity,
refs_bytes: &[u8],
manifest_bytes: &[u8],
limits: DiscoveryLimits,
) -> Result<(RefSet, DiscoveryManifest), graphforge_api::GfError> {
let refs = RefSet::from_json(refs_bytes, limits).map_err(|error| protocol_error(&error))?;
let manifest = DiscoveryManifest::from_json(manifest_bytes, limits)
.map_err(|error| protocol_error(&error))?;
if refs.repository != *identity || manifest.repository != *identity {
return Err(validation("hub.integrity", "discovery repository mismatch"));
}
refs.validate_manifest(&manifest)
.map_err(|error| protocol_error(&error))?;
Ok((refs, manifest))
}
fn read_lineage(
transport: &dyn Transport,
manifest: &DiscoveryManifest,
refs: &RefSet,
limits: DiscoveryLimits,
retry: &RetryPolicy,
) -> Result<(Vec<u8>, ResearchLineage), graphforge_api::GfError> {
let lineage_object = manifest
.lineage_object()
.map_err(|error| protocol_error(&error))?;
let lineage_bytes =
fetch_inventory_object(transport, lineage_object, limits.max_lineage_bytes, retry)?;
let lineage = ResearchLineage::from_json(&lineage_bytes, limits)
.map_err(|error| protocol_error(&error))?;
manifest
.bind_lineage(refs, &lineage)
.map_err(|error| protocol_error(&error))?;
Ok((lineage_bytes, lineage))
}
fn prepare_research_clone(
transport: &dyn Transport,
manifest: &DiscoveryManifest,
refs: &RefSet,
limits: DiscoveryLimits,
args: &CloneArgs,
retry: &RetryPolicy,
) -> Result<Option<(ObjectDescriptor, ResearchCloneContext)>, graphforge_api::GfError> {
let (git_ref, version_uuid, tag) = (
args.git_ref.as_deref(),
args.version_uuid.as_deref(),
args.tag.as_deref(),
);
let (lineage_bytes, lineage) = read_lineage(transport, manifest, refs, limits, retry)?;
let tagged = tag
.map(|name| {
lineage
.tags
.iter()
.find(|tag| tag.name == name)
.map(|tag| tag.version_uuid.clone())
.ok_or_else(|| validation("hub.missing_ref", "tag is absent from research lineage"))
})
.transpose()?;
let version_uuid = version_uuid.or(tagged.as_deref());
let implicit_default = git_ref.is_none() && version_uuid.is_none();
if implicit_default && lineage.branches.is_empty() {
return Ok(None);
}
let git_ref = git_ref.or(implicit_default.then_some(manifest.default_ref.as_str()));
let selected_uuid = resolve_research_version_uuid(&lineage, git_ref, version_uuid)?;
let (version, object) = manifest
.research_version_object(&lineage, &selected_uuid)
.map_err(|error| protocol_error(&error))?;
let advertised_package = version.package.as_ref().ok_or_else(|| {
validation(
"hub.missing_object",
"research Version package is not advertised",
)
})?;
Ok(Some((
object.clone(),
ResearchCloneContext {
selected_branch: git_ref
.filter(|_| version_uuid.is_none())
.and_then(|name| {
lineage
.branches
.iter()
.find(|branch| branch.ref_name == name)
})
.cloned(),
lineage_bytes,
branch_uuid: version.branch_uuid.clone(),
version_uuid: selected_uuid,
version_kind: version.kind.clone(),
identity_digest: version.identity_digest.0.clone(),
package_digest: advertised_package.package_digest.0.clone(),
producing_core_version: advertised_package.producing_core_version.clone(),
},
)))
}
fn staging_path(destination: &Path) -> Result<PathBuf, graphforge_api::GfError> {
let parent = destination.parent().unwrap_or_else(|| Path::new("."));
let name = destination
.file_name()
.and_then(|v| v.to_str())
.ok_or_else(|| {
validation(
"hub.destination_conflict",
"destination must have a UTF-8 name",
)
})?;
Ok(parent.join(format!(".{name}.graphforge-clone")))
}
#[derive(Debug)]
pub(crate) struct CloneStaging {
pub(crate) root: PathBuf,
pub(crate) partial: PathBuf,
project: PathBuf,
operation: PathBuf,
installed: PathBuf,
lock: PathBuf,
_lock: File,
}
impl CloneStaging {
fn new(root: PathBuf, lock: File) -> Self {
Self {
partial: root.join("package.part"),
project: root.join("project"),
operation: root.join("operation"),
installed: root.join("installed.json"),
lock: root.join("clone.lock"),
root,
_lock: lock,
}
}
fn lock_name(&self) -> std::ffi::OsString {
file_name(&self.lock)
}
}
#[cfg(unix)]
pub(crate) fn acquire_staging(destination: &Path) -> Result<CloneStaging, graphforge_api::GfError> {
use std::os::unix::fs::{DirBuilderExt as _, PermissionsExt as _};
let root = staging_path(destination)?;
match std::fs::symlink_metadata(&root) {
Ok(m) if m.file_type().is_symlink() || !m.is_dir() => {
return Err(validation(
"hub.destination_conflict",
"staging path is unsafe",
));
}
Ok(_) => std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700))
.map_err(storage)?,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
let mut b = std::fs::DirBuilder::new();
b.mode(0o700);
b.create(&root).map_err(storage)?;
}
Err(e) => return Err(storage(e)),
}
let lock_path = root.join("clone.lock");
if std::fs::symlink_metadata(&lock_path).is_ok_and(|m| m.file_type().is_symlink()) {
return Err(validation(
"hub.destination_conflict",
"staging lock is unsafe",
));
}
let lock = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.mode(0o600)
.custom_flags(libc::O_NOFOLLOW)
.open(lock_path)
.map_err(storage)?;
match FileExt::try_lock(&lock) {
Ok(()) => {}
Err(TryLockError::WouldBlock) => {
return Err(validation(
"hub.concurrent_clone",
"clone already in progress",
));
}
Err(TryLockError::Error(e)) => return Err(storage(e)),
}
Ok(CloneStaging::new(root, lock))
}
#[cfg(not(unix))]
pub(crate) fn acquire_staging(destination: &Path) -> Result<CloneStaging, graphforge_api::GfError> {
let root = staging_path(destination)?;
match std::fs::symlink_metadata(&root) {
Ok(m) if !m.is_dir() => {
return Err(validation(
"hub.destination_conflict",
"staging path is unsafe",
));
}
Ok(_) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
std::fs::create_dir(&root).map_err(storage)?
}
Err(e) => return Err(storage(e)),
}
let lock = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(root.join("clone.lock"))
.map_err(storage)?;
match FileExt::try_lock(&lock) {
Ok(()) => {}
Err(TryLockError::WouldBlock) => {
return Err(validation(
"hub.concurrent_clone",
"clone already in progress",
));
}
Err(TryLockError::Error(e)) => return Err(storage(e)),
}
Ok(CloneStaging::new(root, lock))
}
#[derive(Serialize, Deserialize)]
struct ResumeState {
digest: String,
length: u64,
location: String,
etag: String,
}
fn save_resume(
checkpoint: &Path,
object: &ObjectDescriptor,
location: &str,
response: &HttpResponse,
) -> Result<(), graphforge_api::GfError> {
let etag = response
.etag
.as_deref()
.filter(|v| v.starts_with('"') && !v.starts_with("W/"))
.ok_or_else(|| validation("hub.integrity", "object response requires a strong ETag"))?;
let state = ResumeState {
digest: object.digest.0.clone(),
length: object.length,
location: location.to_owned(),
etag: etag.to_owned(),
};
reject_unsafe_state_path(checkpoint)?;
let temporary = checkpoint.with_extension("resume.json.tmp");
reject_unsafe_state_path(&temporary)?;
let bytes = serde_json::to_vec(&state).map_err(storage)?;
let mut file = open_private_checkpoint(&temporary)?;
file.write_all(&bytes).map_err(storage)?;
file.sync_all().map_err(storage)?;
drop(file);
std::fs::rename(&temporary, checkpoint).map_err(storage)?;
if let Some(parent) = checkpoint.parent() {
let _ = File::open(parent).and_then(|directory| directory.sync_all());
}
Ok(())
}
fn reject_unsafe_state_path(path: &Path) -> Result<(), graphforge_api::GfError> {
match std::fs::symlink_metadata(path) {
Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_file() => {
Err(validation(
"hub.destination_conflict",
"resume checkpoint path is unsafe",
))
}
Ok(_) => Ok(()),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(storage(error)),
}
}
#[cfg(unix)]
fn open_private_checkpoint(path: &Path) -> Result<File, graphforge_api::GfError> {
OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.mode(0o600)
.custom_flags(libc::O_NOFOLLOW)
.open(path)
.map_err(storage)
}
#[cfg(not(unix))]
fn open_private_checkpoint(path: &Path) -> Result<File, graphforge_api::GfError> {
reject_unsafe_state_path(path)?;
OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.open(path)
.map_err(storage)
}
fn read_resume(checkpoint: &Path) -> Result<Option<ResumeState>, graphforge_api::GfError> {
match std::fs::symlink_metadata(checkpoint) {
Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_file() => {
Err(validation(
"hub.destination_conflict",
"resume checkpoint path is unsafe",
))
}
Ok(_) => {
let file = open_read_nofollow(checkpoint).map_err(storage)?;
let mut bytes = Vec::new();
file.take(64 * 1024)
.read_to_end(&mut bytes)
.map_err(storage)?;
Ok(serde_json::from_slice(&bytes).ok())
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(error) => Err(storage(error)),
}
}
#[cfg(test)]
fn download(
transport: &dyn Transport,
object: &ObjectDescriptor,
partial: &Path,
) -> Result<DownloadReport, graphforge_api::GfError> {
download_with_progress(
transport,
object,
partial,
&mut DownloadReport::default(),
&mut DownloadControl::quiet(&TEST_RETRY_POLICY),
)
.map(|(report, _)| report)
}
#[cfg(any(test, feature = "hub-test-support"))]
const TEST_RETRY_POLICY: RetryPolicy = RetryPolicy {
attempts: RETRY_POLICY.attempts,
initial_backoff: Duration::ZERO,
max_backoff: Duration::ZERO,
};
pub(crate) struct DownloadControl<'a> {
pub(crate) retry: &'a RetryPolicy,
pub(crate) cancelled: &'a AtomicBool,
pub(crate) progress: Option<&'a mut dyn FnMut(u64, u64)>,
}
static NEVER_CANCELLED: AtomicBool = AtomicBool::new(false);
impl<'a> DownloadControl<'a> {
pub(crate) fn quiet(retry: &'a RetryPolicy) -> Self {
Self {
retry,
cancelled: &NEVER_CANCELLED,
progress: None,
}
}
fn report(&mut self, bytes: u64, length: u64) {
if let Some(progress) = self.progress.as_mut() {
progress(bytes, length);
}
}
}
fn cancelled_error() -> graphforge_api::GfError {
validation("hub.interrupted", "clone was cancelled; rerun to resume")
}
#[allow(clippy::too_many_lines)]
fn download_with_progress(
transport: &dyn Transport,
object: &ObjectDescriptor,
partial: &Path,
report: &mut DownloadReport,
control: &mut DownloadControl<'_>,
) -> Result<(DownloadReport, String), graphforge_api::GfError> {
let checkpoint = partial.with_extension("resume.json");
if object.length > MAX_OBJECT_BYTES {
return Err(limit_error("object exceeds the clone byte bound"));
}
let mut offset = match std::fs::symlink_metadata(partial) {
Ok(metadata) if metadata.is_file() && !metadata.file_type().is_symlink() => metadata.len(),
Ok(_) => {
return Err(validation(
"hub.destination_conflict",
"resume path is not a regular file",
));
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => 0,
Err(error) => return Err(storage(error)),
};
if offset > object.length {
open_partial_nofollow(partial, false).map_err(storage)?;
offset = 0;
}
let location = object
.locations
.first()
.ok_or_else(|| validation("hub.missing_object", "portable bundle has no location"))?;
let url = Url::parse(location)
.map_err(|_| validation("hub.unsafe_location", "invalid object URL"))?;
let saved = read_resume(&checkpoint)?;
let mut validator = saved
.filter(|s| {
s.digest == object.digest.0 && s.length == object.length && s.location == *location
})
.map(|s| s.etag);
if offset > 0 && validator.is_none() {
offset = 0;
}
report.resumed_bytes = offset;
let mut first_request = true;
let mut failures = 0_u32;
let mut high_water = offset;
let mut buffer = vec![0_u8; 64 * 1024].into_boxed_slice();
let mut digest = StreamedDigest::default();
control.report(offset, object.length);
while offset < object.length {
if control.cancelled.load(Ordering::Relaxed) {
return Err(cancelled_error());
}
let range = (offset > 0).then_some(offset);
let mut retry_after = None;
let failure = match fetch_once(
transport,
&url,
range,
range.and(validator.as_deref()),
object.length,
&mut report.attempts,
) {
Err(failure) if failure.transient => {
retry_after = failure.retry_after;
failure.error
}
Err(failure) if range.is_some() && failure.status == Some(416) => {
discard_partial(partial, &checkpoint)?;
offset = 0;
validator = None;
if first_request {
report.resumed_bytes = 0;
}
failure.error
}
Err(failure) => return Err(failure.error),
Ok(mut response) => {
let append = if range.is_some() && response.status == 206 {
let expected_range =
format!("bytes {offset}-{}/{}", object.length - 1, object.length);
if response.content_range.as_deref() != Some(expected_range.as_str())
|| response
.etag
.as_deref()
.is_some_and(|etag| Some(etag) != validator.as_deref())
{
let _ = std::fs::remove_file(partial);
let _ = std::fs::remove_file(&checkpoint);
return Err(validation(
"hub.integrity",
"range response does not match the requested object",
));
}
true
} else if response.status == 206 {
return Err(validation(
"hub.integrity",
"unrequested range response for the object",
));
} else {
false
};
if !append {
save_resume(&checkpoint, object, location, &response)?;
validator.clone_from(&response.etag);
offset = 0;
if first_request {
report.resumed_bytes = 0;
}
}
first_request = false;
digest.cover(partial, offset)?;
let mut file = open_partial_nofollow(partial, append).map_err(storage)?;
let mut interrupted = None;
loop {
if control.cancelled.load(Ordering::Relaxed) {
file.sync_all().map_err(storage)?;
return Err(cancelled_error());
}
let Ok(read) = response.body.read(&mut buffer) else {
interrupted = Some(network("object read failed"));
break;
};
if read == 0 {
break;
}
let next = offset
.checked_add(read as u64)
.filter(|next| *next <= object.length)
.ok_or_else(|| limit_error("object exceeds declared size"))?;
file.write_all(&buffer[..read]).map_err(storage)?;
digest.push(&buffer[..read]);
offset = next;
report.transferred_bytes = report.transferred_bytes.saturating_add(read as u64);
control.report(offset, object.length);
}
file.sync_all().map_err(storage)?;
match interrupted {
Some(error) => error,
None if offset < object.length => {
validation("hub.interrupted", "download is incomplete; rerun to resume")
}
None => break,
}
}
};
if offset > high_water {
high_water = offset;
failures = 0;
}
failures += 1;
let Some(wait) = control.retry.wait(failures, retry_after) else {
return Err(resumable_failure(failure, offset));
};
if failures >= control.retry.attempts {
return Err(if offset > 0 {
resumable_failure(failure, offset)
} else {
failure
});
}
std::thread::sleep(wait);
}
let length = open_read_nofollow(partial)
.and_then(|file| file.metadata())
.map_err(storage)?
.len();
if length != object.length {
return Err(validation(
"hub.interrupted",
"download is incomplete; rerun to resume",
));
}
digest.cover(partial, length)?;
let actual = digest.finish();
if actual != object.digest.0 {
let _ = std::fs::remove_file(partial);
let _ = std::fs::remove_file(&checkpoint);
return Err(validation("hub.integrity", "download digest mismatch"));
}
Ok((*report, actual))
}
fn require_downloaded_bytes(
verified: Option<&str>,
downloaded: &str,
) -> Result<(), graphforge_api::GfError> {
if verified == Some(downloaded) {
return Ok(());
}
Err(validation(
"hub.integrity",
"package bytes differ from the downloaded object",
))
}
#[derive(Default)]
struct StreamedDigest {
hash: Option<RunningDigest>,
covered: u64,
}
impl StreamedDigest {
fn push(&mut self, bytes: &[u8]) {
let hash = self.hash.get_or_insert_with(RunningDigest::new);
hash.update(bytes);
self.covered += bytes.len() as u64;
}
fn cover(&mut self, partial: &Path, length: u64) -> Result<(), graphforge_api::GfError> {
if self.covered != length || self.hash.is_none() {
let mut hash = RunningDigest::new();
let file = open_read_nofollow(partial);
match file {
Ok(file) => hash.absorb(&mut file.take(length))?,
Err(error) if length == 0 && error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => return Err(storage(error)),
}
self.hash = Some(hash);
self.covered = length;
}
Ok(())
}
fn finish(self) -> String {
self.hash.unwrap_or_else(RunningDigest::new).finish()
}
}
fn discard_partial(partial: &Path, checkpoint: &Path) -> Result<(), graphforge_api::GfError> {
for path in [partial, checkpoint] {
match std::fs::remove_file(path) {
Ok(()) => {}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => return Err(storage(error)),
}
}
Ok(())
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
struct DownloadReport {
resumed_bytes: u64,
transferred_bytes: u64,
attempts: u32,
}
#[cfg(unix)]
fn open_partial_nofollow(path: &Path, append: bool) -> std::io::Result<File> {
use std::os::unix::fs::OpenOptionsExt as _;
OpenOptions::new()
.create(true)
.write(true)
.append(append)
.truncate(!append)
.custom_flags(libc::O_NOFOLLOW)
.open(path)
}
#[cfg(unix)]
fn open_read_nofollow(path: &Path) -> std::io::Result<File> {
use std::os::unix::fs::OpenOptionsExt as _;
OpenOptions::new()
.read(true)
.custom_flags(libc::O_NOFOLLOW)
.open(path)
}
#[cfg(not(unix))]
fn open_read_nofollow(path: &Path) -> std::io::Result<File> {
let file = File::open(path)?;
if !file.metadata()?.is_file() {
return Err(std::io::Error::other("resume path is not a regular file"));
}
Ok(file)
}
fn ensure_destination_absent(destination: &Path) -> Result<(), graphforge_api::GfError> {
match std::fs::symlink_metadata(destination) {
Ok(_) => Err(validation(
"hub.destination_conflict",
"destination already exists",
)),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(storage(error)),
}
}
#[cfg(not(unix))]
fn open_partial_nofollow(path: &Path, append: bool) -> std::io::Result<File> {
let file = OpenOptions::new()
.create(true)
.write(true)
.append(append)
.truncate(!append)
.open(path)?;
if !file.metadata()?.is_file() {
return Err(std::io::Error::other("resume path is not a regular file"));
}
Ok(file)
}
pub(crate) fn run_clone(
args: CloneArgs,
json: bool,
output: &mut dyn Write,
diagnostics: &mut crate::Diagnostics<'_>,
) -> Result<(), graphforge_api::GfError> {
let transport = crate::auth::CloneTransport::new(
&args.repository,
args.hub.as_deref(),
args.token_env.as_deref(),
args.protection_bypass_env.as_deref(),
)?;
let runtime = clone_telemetry_runtime(args.telemetry_endpoint.as_deref());
let attached = PROCESS_ATTACHED.load(Ordering::Relaxed);
if attached {
install_interrupt_handler();
}
let mut stderr = std::io::stderr();
let mut warn = |warnings: &[graphforge_api::CoreVersionWarning]| diagnostics.warn(warnings);
let mut env = CloneEnv {
retry: RETRY_POLICY,
credential_lookup: |name| std::env::var(name),
cancelled: if attached {
&INTERRUPTED
} else {
&NEVER_CANCELLED
},
available_space,
progress: CloneProgress::new(attached.then_some(&mut stderr as &mut dyn Write)),
warn: Some(&mut warn),
};
let result = run_clone_profiled_with_delays(
&transport,
args,
json,
output,
&runtime,
&CloneDelays::default(),
&mut env,
);
let _ = runtime.shutdown();
result
}
static PROCESS_ATTACHED: AtomicBool = AtomicBool::new(false);
static INTERRUPTED: AtomicBool = AtomicBool::new(false);
pub(crate) fn attach_to_process() {
PROCESS_ATTACHED.store(true, Ordering::Relaxed);
}
fn install_interrupt_handler() {
static INSTALL: std::sync::Once = std::sync::Once::new();
INSTALL.call_once(|| {
let _ = ctrlc::set_handler(|| {
if INTERRUPTED.swap(true, Ordering::SeqCst) {
std::process::exit(130);
}
let _ = writeln!(
std::io::stderr(),
"gf clone: cancelling; a rerun resumes. Press Ctrl-C again to stop now."
);
});
});
}
fn available_space(path: &Path) -> std::io::Result<u64> {
fs4::available_space(path)
}
type WarningSink<'a> = &'a mut dyn FnMut(&[graphforge_api::CoreVersionWarning]);
struct CloneEnv<'a> {
credential_lookup: fn(&str) -> Result<String, std::env::VarError>,
retry: RetryPolicy,
cancelled: &'a AtomicBool,
available_space: fn(&Path) -> std::io::Result<u64>,
progress: CloneProgress<'a>,
warn: Option<WarningSink<'a>>,
}
#[cfg(any(test, feature = "hub-test-support"))]
impl CloneEnv<'_> {
fn quiet<'a>() -> CloneEnv<'a> {
CloneEnv {
retry: TEST_RETRY_POLICY,
credential_lookup: |name| std::env::var(name),
cancelled: &NEVER_CANCELLED,
available_space,
progress: CloneProgress::new(None),
warn: None,
}
}
}
const PROGRESS_INTERVAL: Duration = Duration::from_secs(1);
struct CloneProgress<'a> {
sink: Option<&'a mut dyn Write>,
last: Option<Instant>,
}
impl<'a> CloneProgress<'a> {
fn new(sink: Option<&'a mut dyn Write>) -> Self {
Self { sink, last: None }
}
fn phase(&mut self, message: &str) {
if let Some(sink) = self.sink.as_mut() {
let _ = writeln!(sink, "gf clone: {message}");
}
}
fn bytes(&mut self, done: u64, total: u64) {
let Some(sink) = self.sink.as_mut() else {
return;
};
let now = Instant::now();
if done < total
&& self
.last
.is_some_and(|last| now.duration_since(last) < PROGRESS_INTERVAL)
{
return;
}
self.last = Some(now);
let percent = if total == 0 {
100
} else {
u128::from(done) * 100 / u128::from(total)
};
let _ = writeln!(
sink,
"gf clone: downloaded {} of {} ({percent}%)",
human_bytes(done),
human_bytes(total)
);
}
}
fn human_bytes(bytes: u64) -> String {
const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"];
let mut unit = 0;
let mut scale = 1_u64;
while unit + 1 < UNITS.len() && bytes / scale >= 1024 {
scale *= 1024;
unit += 1;
}
if unit == 0 {
return format!("{bytes} B");
}
let tenths = u128::from(bytes) * 10 / u128::from(scale);
format!("{}.{} {}", tenths / 10, tenths % 10, UNITS[unit])
}
fn required_space(
length: u64,
downloaded: u64,
staged_import: u64,
committed: bool,
research: bool,
) -> u64 {
let footprint = PortableImportFootprint::of_package_length(length, research);
let import_peak = if committed {
footprint.replay_peak_bytes()
} else {
footprint.peak_bytes()
};
let verification = footprint.verification_peak_bytes();
let import = import_peak.saturating_sub(staged_import);
length
.saturating_sub(downloaded)
.saturating_add(verification.max(import))
}
fn check_free_space(
staging: &CloneStaging,
length: u64,
committed: bool,
research: bool,
operation: &OperationId,
available: fn(&Path) -> std::io::Result<u64>,
) -> Result<(), graphforge_api::GfError> {
let downloaded = std::fs::symlink_metadata(&staging.partial)
.ok()
.filter(std::fs::Metadata::is_file)
.map_or(0, |metadata| metadata.len().min(length));
let required = required_space(
length,
downloaded,
graphforge_api::portable_import_held_bytes(
&staging.project,
operation,
PortableLimits::default(),
),
committed,
research,
);
let available = available(&staging.root).map_err(storage)?;
if available < required {
return Err(graphforge_api::GfError::Storage(format!(
"hub.insufficient_space: the destination filesystem has {} available; \
this clone needs about {}",
human_bytes(available),
human_bytes(required)
)));
}
Ok(())
}
fn clone_operation(
identity: &RepositoryIdentity,
immutable_version: &str,
research: Option<&ResearchCloneContext>,
) -> OperationId {
OperationId(match research {
None => graphforge_api::hub_clone_operation(&canonical_name(identity), immutable_version),
Some(context) => graphforge_api::hub_research_clone_operation(
&canonical_name(identity),
immutable_version,
&context.version_uuid,
&context.identity_digest,
),
})
}
fn import_committed(staging: &CloneStaging, operation: &OperationId) -> bool {
if std::fs::symlink_metadata(&staging.project).is_err() {
return false;
}
let expected = graphforge_api::portable_import_generation(&operation.0);
graphforge_api::read_committed_generation_identity(&staging.project)
.is_ok_and(|identity| identity.generation_uuid == expected)
}
fn discards_failed_import(
staging: &CloneStaging,
operation: &OperationId,
error: &graphforge_api::PortableError,
) -> bool {
error.committed_import.is_none() && !import_committed(staging, operation)
}
fn remove_stale_scratch(staging: &CloneStaging) -> Result<(), graphforge_api::GfError> {
for entry in std::fs::read_dir(&staging.root).map_err(storage)? {
let entry = entry.map_err(storage)?;
if entry
.file_name()
.to_string_lossy()
.starts_with(RESEARCH_SCRATCH_PREFIX)
&& entry.file_type().map_err(storage)?.is_dir()
{
std::fs::remove_dir_all(entry.path()).map_err(storage)?;
}
}
Ok(())
}
const RESEARCH_SCRATCH_PREFIX: &str = ".graphforge-research-validation-";
fn clone_telemetry_runtime(endpoint: Option<&str>) -> TelemetryRuntime {
let Some(endpoint) = endpoint else {
return TelemetryRuntime::default();
};
TelemetryRuntime::new(TelemetryConfig {
mode: TelemetryMode::OtlpHttpJson,
otlp: Some(OtlpConfig {
endpoint: endpoint.to_owned(),
headers: BTreeMap::default(),
}),
..TelemetryConfig::default()
})
.unwrap_or_default()
}
#[cfg(any(test, feature = "hub-test-support"))]
pub(crate) fn run_clone_with(
transport: &dyn Transport,
args: CloneArgs,
json: bool,
output: &mut dyn Write,
) -> Result<(), graphforge_api::GfError> {
run_clone_profiled(transport, args, json, output, &TelemetryRuntime::default())
}
#[cfg(any(test, feature = "hub-test-support"))]
fn run_clone_profiled(
transport: &dyn Transport,
args: CloneArgs,
json: bool,
output: &mut dyn Write,
runtime: &TelemetryRuntime,
) -> Result<(), graphforge_api::GfError> {
run_clone_profiled_with_delays(
transport,
args,
json,
output,
runtime,
&CloneDelays::default(),
&mut CloneEnv::quiet(),
)
}
#[derive(Default)]
struct CloneDelays {
clock: CloneClock,
verification: Duration,
import: Duration,
reopen: Duration,
#[cfg(test)]
before_import: Option<Box<dyn Fn()>>,
}
fn run_clone_profiled_with_delays(
transport: &dyn Transport,
args: CloneArgs,
json: bool,
output: &mut dyn Write,
runtime: &TelemetryRuntime,
delays: &CloneDelays,
env: &mut CloneEnv<'_>,
) -> Result<(), graphforge_api::GfError> {
let mut profile = CloneProfile::new(runtime);
#[cfg(test)]
{
profile.clock = delays.clock.clone();
}
let result = run_clone_job(transport, args, &mut profile, delays, env)
.and_then(|result| write_clone_result(&result, json, output));
profile.finish(&result);
result
}
#[cfg(test)]
fn clone_failpoint(name: &str) {
if std::env::var("GRAPHFORGE_CLONE_FAILPOINT").as_deref() == Ok(name) {
std::process::exit(CLONE_FAILPOINT_EXIT);
}
}
#[cfg(test)]
const CLONE_FAILPOINT_EXIT: i32 = 86;
#[cfg(not(test))]
const fn clone_failpoint(_name: &str) {}
#[derive(Serialize, Deserialize)]
struct InstallRecord {
generation_uuid: String,
result: CloneResult,
}
fn finish_installed_clone(
destination: &Path,
) -> Result<Option<CloneResult>, graphforge_api::GfError> {
let root = staging_path(destination)?;
match std::fs::symlink_metadata(&root) {
Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => {}
_ => return Ok(None),
}
let staging = acquire_staging(destination)?;
let record = if std::fs::symlink_metadata(&staging.project).is_ok() {
None
} else {
read_install_record(&staging.installed)?
};
let finished = record.filter(|record| {
destination
.to_str()
.and_then(|path| GraphForge::new(Some(path)).ok())
.and_then(|graph| graph.committed_generation_identity().ok())
.is_some_and(|identity| identity.generation_uuid.to_string() == record.generation_uuid)
});
release_staging(staging);
Ok(finished.map(|record| record.result))
}
fn read_install_record(path: &Path) -> Result<Option<InstallRecord>, graphforge_api::GfError> {
match std::fs::symlink_metadata(path) {
Ok(metadata) if metadata.is_file() && !metadata.file_type().is_symlink() => {
let mut bytes = Vec::new();
open_read_nofollow(path)
.map_err(storage)?
.take(64 * 1024)
.read_to_end(&mut bytes)
.map_err(storage)?;
Ok(serde_json::from_slice(&bytes).ok())
}
Ok(_) => Err(validation(
"hub.destination_conflict",
"install record path is unsafe",
)),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(error) => Err(storage(error)),
}
}
fn write_staging_file(path: &Path, bytes: &[u8]) -> Result<(), graphforge_api::GfError> {
reject_unsafe_state_path(path)?;
let temporary = path.with_extension("tmp");
reject_unsafe_state_path(&temporary)?;
let mut file = open_private_checkpoint(&temporary)?;
file.write_all(bytes).map_err(storage)?;
file.sync_all().map_err(storage)?;
drop(file);
std::fs::rename(&temporary, path).map_err(storage)?;
if let Some(parent) = path.parent() {
let _ = File::open(parent).and_then(|directory| directory.sync_all());
}
Ok(())
}
fn bind_import_target(
staging: &CloneStaging,
operation: &OperationId,
) -> Result<(), graphforge_api::GfError> {
let expected = operation.0.hyphenated().to_string();
let recorded = read_import_operation(staging)?;
if recorded.as_deref() == Some(expected.as_bytes()) {
return Ok(());
}
clear_import_target(staging)?;
write_staging_file(&staging.operation, expected.as_bytes())
}
fn read_import_operation(
staging: &CloneStaging,
) -> Result<Option<Vec<u8>>, graphforge_api::GfError> {
Ok(match std::fs::symlink_metadata(&staging.operation) {
Ok(metadata) if metadata.is_file() && !metadata.file_type().is_symlink() => {
let mut bytes = Vec::new();
open_read_nofollow(&staging.operation)
.map_err(storage)?
.take(128)
.read_to_end(&mut bytes)
.map_err(storage)?;
Some(bytes)
}
Ok(_) => {
return Err(validation(
"hub.destination_conflict",
"staged operation path is unsafe",
));
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
Err(error) => return Err(storage(error)),
})
}
fn clear_import_target(staging: &CloneStaging) -> Result<(), graphforge_api::GfError> {
let keep = [
staging.lock_name(),
file_name(&staging.partial),
file_name(&staging.partial.with_extension("resume.json")),
];
for entry in std::fs::read_dir(&staging.root).map_err(storage)? {
let entry = entry.map_err(storage)?;
if keep.iter().any(|name| *name == entry.file_name()) {
continue;
}
let path = entry.path();
if entry.file_type().map_err(storage)?.is_dir() {
std::fs::remove_dir_all(&path).map_err(storage)?;
} else {
std::fs::remove_file(&path).map_err(storage)?;
}
}
Ok(())
}
fn file_name(path: &Path) -> std::ffi::OsString {
path.file_name().map(ToOwned::to_owned).unwrap_or_default()
}
fn install_destination(
staging: &CloneStaging,
destination: &Path,
) -> Result<(), graphforge_api::GfError> {
match graphforge_filesystem::rename_no_replace(&staging.project, destination) {
Ok(()) => {}
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
return Err(validation(
"hub.destination_conflict",
"destination already exists",
));
}
Err(error) => return Err(storage(error)),
}
if let Some(parent) = destination
.parent()
.filter(|parent| !parent.as_os_str().is_empty())
{
let _ = File::open(parent).and_then(|directory| directory.sync_all());
}
Ok(())
}
fn import_error(error: &graphforge_api::PortableError) -> graphforge_api::GfError {
let code = portable_code(error.code);
let mut message = format!("{code}: portable project import failed: {error}");
if let Some(cause) = &error.cause {
let _ = write!(message, " (cause: {cause})");
}
match error.code {
PortableErrorCode::UnsupportedFilesystem => graphforge_api::GfError::Project {
code: graphforge_api::ProjectErrorCode::UnsupportedFilesystem,
message,
},
PortableErrorCode::Io
| PortableErrorCode::Cancelled
| PortableErrorCode::InsufficientSpace => graphforge_api::GfError::Storage(message),
_ => graphforge_api::GfError::Validation(message),
}
}
fn portable_code(code: PortableErrorCode) -> &'static str {
match code {
PortableErrorCode::Cancelled => "hub.package.cancelled",
PortableErrorCode::LimitExceeded => "hub.package.limit_exceeded",
PortableErrorCode::Io => "hub.package.io",
PortableErrorCode::InvalidStructure => "hub.package.invalid_structure",
PortableErrorCode::InvalidPath => "hub.package.invalid_path",
PortableErrorCode::DuplicateEntry => "hub.package.duplicate_entry",
PortableErrorCode::UnsupportedProjectFormat => "hub.package.unsupported_format",
PortableErrorCode::UnsupportedFuture => "hub.package.unsupported_future",
PortableErrorCode::Incompatible => "hub.package.incompatible",
PortableErrorCode::DigestMismatch => "hub.package.digest_mismatch",
PortableErrorCode::ConcurrentMutation => "hub.package.concurrent_mutation",
PortableErrorCode::UnsupportedFilesystem => "GF_UNSUPPORTED_FILESYSTEM",
PortableErrorCode::InsufficientSpace => "hub.insufficient_space",
}
}
fn release_staging_if_unused(staging: CloneStaging) {
let reusable = std::fs::read_dir(&staging.root).is_ok_and(|entries| {
entries
.flatten()
.any(|entry| entry.file_name() != staging.lock_name())
});
if reusable {
drop(staging);
} else {
release_staging(staging);
}
}
pub(crate) fn release_staging(staging: CloneStaging) {
clear_staging(&staging);
let root = staging.root.clone();
drop(staging);
remove_staging_dir(&root);
}
fn remove_staging_dir(root: &Path) {
let _ = std::fs::remove_dir(root);
}
fn clear_staging(staging: &CloneStaging) {
let Ok(entries) = std::fs::read_dir(&staging.root) else {
return;
};
let entries: Vec<_> = entries
.flatten()
.map(|entry| {
let directory = entry.file_type().is_ok_and(|kind| kind.is_dir());
(entry.path(), directory)
})
.collect();
for (path, directory) in removal_order(staging, entries) {
let _ = if directory {
std::fs::remove_dir_all(&path)
} else {
std::fs::remove_file(&path)
};
}
}
fn removal_order(
staging: &CloneStaging,
mut entries: Vec<(PathBuf, bool)>,
) -> Vec<(PathBuf, bool)> {
entries.sort_by_key(|(path, _)| (*path == staging.lock, *path == staging.installed));
entries
}
fn resumable_failure(failure: graphforge_api::GfError, offset: u64) -> graphforge_api::GfError {
const GUIDANCE: &str = "download is incomplete; rerun to resume";
match failure {
graphforge_api::GfError::Storage(detail)
if offset > 0
&& (detail.starts_with("hub.unavailable:")
|| detail.starts_with("hub.rate_limited:")) =>
{
graphforge_api::GfError::Storage(format!("{detail}; {GUIDANCE}"))
}
other if offset == 0 => other,
_ => validation("hub.interrupted", GUIDANCE),
}
}
#[allow(clippy::too_many_lines)]
fn run_clone_job(
transport: &dyn Transport,
mut args: CloneArgs,
profile: &mut CloneProfile<'_>,
delays: &CloneDelays,
env: &mut CloneEnv<'_>,
) -> Result<CloneResult, graphforge_api::GfError> {
if [
args.git_ref.is_some(),
args.version_uuid.is_some(),
args.tag.is_some(),
]
.iter()
.filter(|given| **given)
.count()
> 1
{
return Err(validation(
"hub.invalid_identity",
"specify only one of --ref, --version-uuid and --tag",
));
}
let requested_destination = args.destination.take();
let (identity, base, destination, finished) = profile.stage(
Stage::IdentityValidation,
ComponentKind::Cli,
ComponentRole::Facade,
None,
1,
|| {
let (identity, base) = parse_input_at(&args.repository, args.hub.as_deref())?;
let destination =
requested_destination.unwrap_or_else(|| PathBuf::from(&identity.repository));
if std::fs::symlink_metadata(&destination).is_ok() {
let finished = finish_installed_clone(&destination)?.ok_or_else(|| {
validation("hub.destination_conflict", "destination already exists")
})?;
return Ok(((identity, base, destination, Some(finished)), None, None));
}
ensure_destination_absent(&destination)?;
graphforge_api::filesystem_durability_preflight(&destination)?;
Ok(((identity, base, destination, None), None, None))
},
)?;
if let Some(finished) = finished {
return Ok(finished);
}
let credentials = DiscoveryCredentials::load(
&base,
args.token_env.as_deref(),
args.protection_bypass_env.as_deref(),
env.credential_lookup,
)?;
let move_transport = crate::hub_http::RepositoryMoveTransport::new(transport, &identity, &base)
.with_credentials(credentials.as_ref());
let transport = &move_transport as &dyn Transport;
profile.handoff(
ComponentKind::Cli,
ComponentKind::NetworkTransport,
HandoffKind::Call,
None,
);
let mut refs_attempts = 0;
let refs_result = profile.stage(
Stage::RefsDiscovery,
ComponentKind::NetworkTransport,
ComponentRole::Transfer,
Some(WaitReason::Network),
1,
|| {
let bytes = read_bounded(
fetch_discovery_with_attempts(
transport,
&endpoint(&base, "refs"),
credentials.as_ref(),
MAX_METADATA_BYTES as u64,
&mut refs_attempts,
&env.retry,
)?,
MAX_METADATA_BYTES,
)?;
let length = bytes.len() as u64;
Ok((bytes, Some(length), None))
},
);
if let Some(stage) = profile.stages.last_mut() {
stage.attempt = refs_attempts
.saturating_add(move_transport.extra_metadata_requests())
.max(1);
}
let refs_bytes = refs_result?;
let (_, manifest_base, _) = move_transport.resolved(false)?;
let mut manifest_attempts = 0;
let manifest_result = profile.stage(
Stage::ManifestDiscovery,
ComponentKind::NetworkTransport,
ComponentRole::Transfer,
Some(WaitReason::Network),
1,
|| {
let bytes = read_bounded(
fetch_discovery_with_attempts(
transport,
&endpoint(&manifest_base, "manifest"),
credentials.as_ref(),
MAX_METADATA_BYTES as u64,
&mut manifest_attempts,
&env.retry,
)?,
MAX_METADATA_BYTES,
)?;
let length = bytes.len() as u64;
Ok((bytes, Some(length), None))
},
);
if let Some(stage) = profile.stages.last_mut() {
stage.attempt = manifest_attempts.max(1);
}
let manifest_bytes = manifest_result?;
let (identity, _, moved) = move_transport.resolved(false)?;
let limits = discovery_limits();
profile.handoff(
ComponentKind::NetworkTransport,
ComponentKind::Discovery,
HandoffKind::Return,
Some((refs_bytes.len() + manifest_bytes.len()) as u64),
);
let (manifest, staging) = profile.stage(
Stage::ManifestDiscovery,
ComponentKind::Discovery,
ComponentRole::Verification,
None,
1,
|| {
let (_, manifest) = verify_discovery(&identity, &refs_bytes, &manifest_bytes, limits)?;
let staging = acquire_staging(&destination)?;
Ok(((manifest, staging), None, None))
},
)?;
let discovered = Discovered {
base: &manifest_base,
identity: &identity,
moved: moved.as_ref(),
destination: &destination,
refs_bytes: &refs_bytes,
manifest_bytes: &manifest_bytes,
manifest: &manifest,
limits,
};
let result = clone_into_staging(
transport,
&args,
&discovered,
&staging,
profile,
delays,
env,
);
match result {
Ok(result) => {
profile.stage(
Stage::Cleanup,
ComponentKind::Storage,
ComponentRole::Persistence,
None,
1,
|| {
release_staging(staging);
Ok(((), None, None))
},
)?;
Ok(result)
}
Err(error) => {
if std::fs::symlink_metadata(&destination).is_ok() {
release_staging(staging);
} else {
release_staging_if_unused(staging);
}
Err(error)
}
}
}
struct Discovered<'a> {
base: &'a Url,
identity: &'a RepositoryIdentity,
moved: Option<&'a graphforge_discovery::RepositoryMoved>,
destination: &'a Path,
refs_bytes: &'a [u8],
manifest_bytes: &'a [u8],
manifest: &'a DiscoveryManifest,
limits: DiscoveryLimits,
}
#[allow(clippy::too_many_lines)]
fn clone_into_staging(
transport: &dyn Transport,
args: &CloneArgs,
discovered: &Discovered<'_>,
staging: &CloneStaging,
profile: &mut CloneProfile<'_>,
delays: &CloneDelays,
env: &mut CloneEnv<'_>,
) -> Result<CloneResult, graphforge_api::GfError> {
let Discovered {
base,
identity,
moved,
destination,
refs_bytes,
manifest_bytes,
manifest,
limits,
} = *discovered;
let refs = RefSet::from_json(refs_bytes, limits).map_err(|error| protocol_error(&error))?;
let research_requested =
args.git_ref.is_some() || args.version_uuid.is_some() || args.tag.is_some();
let (object, research_context) = if research_requested || manifest.lineage.is_some() {
if manifest.lineage.is_none() {
return Err(validation(
"hub.missing_object",
"research clone requires an advertised lineage document",
));
}
profile.handoff(
ComponentKind::Discovery,
ComponentKind::NetworkTransport,
HandoffKind::Call,
None,
);
let prepared =
prepare_research_clone(transport, manifest, &refs, limits, args, &env.retry)?;
profile.handoff(
ComponentKind::NetworkTransport,
ComponentKind::Discovery,
HandoffKind::Return,
prepared
.as_ref()
.map(|(_, context)| context.lineage_bytes.len() as u64),
);
if let Some((object, context)) = prepared {
(object, Some(context))
} else {
(select_bundle(manifest)?.clone(), None)
}
} else {
(select_bundle(manifest)?.clone(), None)
};
if object.length > MAX_OBJECT_BYTES {
return Err(limit_error("object exceeds the clone byte bound"));
}
let producing_core_version = research_context.as_ref().map_or(
manifest.package.producing_core_version.as_str(),
|context| context.producing_core_version.as_str(),
);
let warnings: Vec<_> = graphforge_api::check_core_version(Some(producing_core_version))
.into_iter()
.collect();
if !warnings.is_empty()
&& let Some(warn) = env.warn.as_mut()
{
warn(&warnings);
}
remove_stale_scratch(staging)?;
let planned = clone_operation(
identity,
&manifest.immutable_version.0,
research_context.as_ref(),
);
check_free_space(
staging,
object.length,
import_committed(staging, &planned),
research_context.is_some(),
&planned,
env.available_space,
)?;
let partial = staging.partial.clone();
profile.handoff(
ComponentKind::Discovery,
ComponentKind::NetworkTransport,
HandoffKind::Call,
None,
);
let mut download_progress = DownloadReport::default();
let download_result = profile.stage(
Stage::Download,
ComponentKind::NetworkTransport,
ComponentRole::Transfer,
Some(WaitReason::Network),
1,
|| {
let CloneEnv {
retry,
cancelled,
progress,
..
} = env;
let mut report_bytes = |done, total| progress.bytes(done, total);
let (report, digest) = download_with_progress(
transport,
&object,
&partial,
&mut download_progress,
&mut DownloadControl {
retry,
cancelled,
progress: Some(&mut report_bytes),
},
)?;
Ok(((report, digest), Some(report.transferred_bytes), None))
},
);
if let Some(stage) = profile.stages.last_mut() {
stage.attempt = download_progress.attempts.max(1);
stage.bytes = Some(download_progress.transferred_bytes);
stage.resumed_bytes = Some(download_progress.resumed_bytes);
}
let (download, download_digest) = download_result?;
clone_failpoint("clone.after_download");
let portable_limits = PortableLimits::default();
profile.handoff(
ComponentKind::NetworkTransport,
ComponentKind::PortableVerify,
HandoffKind::Transfer,
Some(download.resumed_bytes + download.transferred_bytes),
);
env.progress.phase("verifying the package");
let cancelled = env.cancelled;
let verified = profile.stage(
Stage::PortableVerification,
ComponentKind::PortableVerify,
ComponentRole::Verification,
None,
1,
|| {
delays.clock.wait(delays.verification);
let outcome = if let Some(context) = &research_context {
let verified_research =
verify_discovered_research_version(&DiscoveryResearchVersionRequest {
manifest_json: manifest_bytes,
refs_json: refs_bytes,
lineage_json: &context.lineage_bytes,
expected_repository: identity,
version_uuid: &context.version_uuid,
package: &partial,
discovery_limits: limits,
portable_limits,
mode: PortableMode::Full,
cancelled: Some(cancelled),
scratch: Some(&staging.root),
})
.map_err(research_version_error)?;
require_downloaded_bytes(
verified_research.report.transport_digest.as_deref(),
&download_digest,
)?;
(
manifest.immutable_version.0.clone(),
Some(context.version_uuid.clone()),
Some(context.version_kind.clone()),
)
} else {
let accepted = verify_discovered_portable(&DiscoveryPortableRequest {
manifest_json: manifest_bytes,
refs_json: refs_bytes,
expected_repository: identity,
package: &partial,
discovery_limits: limits,
portable_limits,
mode: PortableMode::Full,
cancelled: Some(cancelled),
})
.map_err(portable_error)?;
require_downloaded_bytes(
accepted.report.transport_digest.as_deref(),
&download_digest,
)?;
(accepted.immutable_version, None, None)
};
Ok((outcome, Some(object.length), None))
},
)?;
let (immutable_version, research_version_uuid, research_version_kind) = verified;
let operation_id = clone_operation(identity, &immutable_version, research_context.as_ref());
profile.handoff(
ComponentKind::PortableVerify,
ComponentKind::Api,
HandoffKind::Call,
Some(object.length),
);
profile.handoff(
ComponentKind::Api,
ComponentKind::PortableImport,
HandoffKind::Call,
Some(object.length),
);
env.progress.phase("importing the project");
let mut imported = profile.stage(
Stage::AtomicImport,
ComponentKind::PortableImport,
ComponentRole::Persistence,
None,
1,
|| {
delays.clock.wait(delays.import);
#[cfg(test)]
if let Some(hook) = &delays.before_import {
hook();
}
let existing_project = match std::fs::symlink_metadata(&staging.project) {
Ok(_) => true,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => false,
Err(error) => return Err(storage(error)),
};
if let Some(context) = &research_context
&& existing_project
{
if read_import_operation(staging)?.as_deref()
!= Some(operation_id.0.hyphenated().to_string().as_bytes())
{
return Err(validation(
"hub.destination_conflict",
"research staging has another import operation",
));
}
let lineage = ResearchLineage::from_json(&context.lineage_bytes, limits)
.map_err(|e| protocol_error(&e))?;
if context.selected_branch.is_none()
&& let Some(generation_uuid) = GraphForge::continue_imported_research_tags(
&staging.project,
&lineage,
uuid::Uuid::parse_str(&context.version_uuid).map_err(storage)?,
operation_id.0,
)?
{
return Ok((
CloneImported {
package_digest: context.package_digest.clone(),
generation_uuid,
},
None,
None,
));
}
if let Some(branch) = &context.selected_branch {
let receipt = GraphForge::continue_imported_research_branch(
&staging.project,
branch,
&lineage,
&format!(
"{}/{}",
base.origin().ascii_serialization(),
identity.canonical_name()
),
operation_id.0,
)?;
if let Some(receipt) = receipt {
return Ok((
CloneImported {
package_digest: context.package_digest.clone(),
generation_uuid: receipt.generation_uuid,
},
None,
None,
));
}
}
}
bind_import_target(staging, &operation_id)?;
GraphForge::import_portable(
&staging.project,
&PortableImportRequest {
input: partial.clone(),
operation_id,
limits: portable_limits,
},
Some(cancelled),
)
.map_err(|error| {
if research_context.is_none()
&& discards_failed_import(staging, &operation_id, &error)
&& let Err(cleanup) = clear_import_target(staging)
{
return graphforge_api::GfError::Storage(format!(
"{}: portable project import failed ({error}), and removing \
its staged target failed: {cleanup}",
portable_code(error.code)
));
}
import_error(&error)
})
.map(|imported| {
(
CloneImported {
package_digest: imported.package_digest,
generation_uuid: imported.generation_uuid,
},
Some(object.length),
None,
)
})
},
)?;
if let Some(context) = &research_context
&& imported.package_digest != context.package_digest
{
return Err(validation(
"hub.package.package_digest_mismatch",
"imported research package differs from the verified Version package",
));
}
if let Some(branch) = research_context
.as_ref()
.and_then(|context| context.selected_branch.as_ref())
{
let lineage = ResearchLineage::from_json(
&research_context
.as_ref()
.expect("selected context")
.lineage_bytes,
limits,
)
.map_err(|e| protocol_error(&e))?;
clone_failpoint("clone.before_branch_continuation");
let receipt = GraphForge::continue_imported_research_branch(
&staging.project,
branch,
&lineage,
&format!(
"{}/{}",
base.origin().ascii_serialization(),
identity.canonical_name()
),
operation_id.0,
)?
.ok_or_else(|| {
validation(
"hub.destination_conflict",
"import did not commit its selected Branch",
)
})?;
imported.generation_uuid = receipt.generation_uuid;
clone_failpoint("clone.after_branch_continuation");
}
if let Some(context) = &research_context
&& context.selected_branch.is_none()
{
let lineage = ResearchLineage::from_json(&context.lineage_bytes, limits)
.map_err(|error| protocol_error(&error))?;
imported.generation_uuid = GraphForge::continue_imported_research_tags(
&staging.project,
&lineage,
uuid::Uuid::parse_str(&context.version_uuid).map_err(storage)?,
operation_id.0,
)?
.ok_or_else(|| validation("hub.destination_conflict", "Version import is unfinished"))?;
clone_failpoint("clone.after_version_tags");
}
profile.handoff(
ComponentKind::PortableImport,
ComponentKind::Storage,
HandoffKind::Write,
Some(object.length),
);
profile.handoff(
ComponentKind::Storage,
ComponentKind::Publication,
HandoffKind::Write,
Some(object.length),
);
profile.handoff(
ComponentKind::Publication,
ComponentKind::Api,
HandoffKind::Return,
None,
);
profile.handoff(
ComponentKind::Api,
ComponentKind::Recovery,
HandoffKind::Return,
None,
);
profile.stage(
Stage::Reopen,
ComponentKind::Recovery,
ComponentRole::Verification,
None,
1,
|| {
delays.clock.wait(delays.reopen);
Ok(((), None, None))
},
)?;
profile.handoff(
ComponentKind::Recovery,
ComponentKind::Storage,
HandoffKind::Call,
None,
);
let result = CloneResult {
contract: CLONE_CONTRACT.to_owned(),
repository: canonical_name(identity),
destination: destination.display().to_string(),
immutable_version,
package_digest: imported.package_digest,
generation_uuid: imported.generation_uuid.to_string(),
resumed_bytes: download.resumed_bytes,
resolved_ref: research_context
.as_ref()
.and_then(|context| context.selected_branch.as_ref())
.map(|branch| branch.ref_name.clone()),
research_branch_uuid: research_context
.as_ref()
.map(|context| context.branch_uuid.clone()),
research_version_uuid,
research_version_kind,
research_version_identity: research_context
.as_ref()
.map(|context| context.identity_digest.clone()),
moved_from: moved.map(|m| m.moved_from.canonical_name()),
moved_to: moved.map(|m| m.moved_to.canonical_name()),
warnings,
};
let record = InstallRecord {
generation_uuid: result.generation_uuid.clone(),
result,
};
write_staging_file(
&staging.installed,
&serde_json::to_vec(&record).map_err(storage)?,
)?;
clone_failpoint("clone.before_install");
install_destination(staging, destination)?;
clone_failpoint("clone.after_install");
Ok(record.result)
}
fn write_clone_result(
result: &CloneResult,
json: bool,
output: &mut dyn Write,
) -> Result<(), graphforge_api::GfError> {
if json {
serde_json::to_writer(&mut *output, &result)
.map_err(|e| graphforge_api::GfError::Execution(e.to_string()))?;
writeln!(output).map_err(storage)?;
} else {
if let (Some(from), Some(to)) = (&result.moved_from, &result.moved_to) {
writeln!(output, "Repository moved from {from} to {to}").map_err(storage)?;
}
let selected = match (&result.resolved_ref, &result.research_version_uuid) {
(Some(name), Some(version)) => format!(" ref {name} (Version {version})"),
(None, Some(version)) => format!(" Version {version}"),
_ => String::new(),
};
writeln!(
output,
"Cloned {}{selected} to {}",
result.repository, result.destination
)
.map_err(storage)?;
}
Ok(())
}
fn canonical_name(identity: &RepositoryIdentity) -> String {
identity.canonical_name()
}
fn protocol_error(error: &DiscoveryError) -> graphforge_api::GfError {
let code = match error.code {
DiscoveryErrorCode::InvalidIdentity => "hub.invalid_identity",
DiscoveryErrorCode::MalformedResponse => "hub.malformed_response",
DiscoveryErrorCode::UnsupportedFuture => "hub.unsupported_future",
DiscoveryErrorCode::MissingRef => "hub.missing_ref",
DiscoveryErrorCode::MissingObject | DiscoveryErrorCode::Gone => "hub.missing_object",
DiscoveryErrorCode::AmbiguousIdentity => "hub.ambiguous_identity",
DiscoveryErrorCode::IntegrityFailure => "hub.integrity_failure",
DiscoveryErrorCode::UnsafeLocation => "hub.unsafe_location",
DiscoveryErrorCode::LimitExceeded => "hub.limit_exceeded",
DiscoveryErrorCode::Duplicate => "hub.duplicate",
};
validation(code, error.detail())
}
fn research_version_error(error: DiscoveryResearchVersionError) -> graphforge_api::GfError {
match error {
DiscoveryResearchVersionError::Discovery(error) => protocol_error(&error),
DiscoveryResearchVersionError::ReferenceMismatch(mismatch) => validation(
match mismatch {
DiscoveryPortableMismatch::Repository => "hub.package.repository_mismatch",
DiscoveryPortableMismatch::ImmutableVersion => {
"hub.package.immutable_version_mismatch"
}
DiscoveryPortableMismatch::PackageDigest => "hub.package.package_digest_mismatch",
DiscoveryPortableMismatch::ProducingCoreVersion => {
"hub.package.producing_core_version_mismatch"
}
DiscoveryPortableMismatch::ModuleIdentity => "hub.module.identity_mismatch",
DiscoveryPortableMismatch::ModuleContentDigest => {
"hub.module.content_digest_mismatch"
}
DiscoveryPortableMismatch::ResearchVersionIdentity => {
"hub.package.research_version_mismatch"
}
},
"research discovery reference mismatch",
),
DiscoveryResearchVersionError::Portable(error) => validation(
portable_code(error.code),
"research portable verification failed",
),
}
}
fn portable_error(error: DiscoveryPortableError) -> graphforge_api::GfError {
match error {
DiscoveryPortableError::Discovery(error) => protocol_error(&error),
DiscoveryPortableError::ReferenceMismatch(mismatch) => validation(
match mismatch {
DiscoveryPortableMismatch::Repository => "hub.package.repository_mismatch",
DiscoveryPortableMismatch::ImmutableVersion => {
"hub.package.immutable_version_mismatch"
}
DiscoveryPortableMismatch::PackageDigest => "hub.package.package_digest_mismatch",
DiscoveryPortableMismatch::ProducingCoreVersion => {
"hub.package.producing_core_version_mismatch"
}
DiscoveryPortableMismatch::ModuleIdentity => "hub.module.identity_mismatch",
DiscoveryPortableMismatch::ModuleContentDigest => {
"hub.module.content_digest_mismatch"
}
DiscoveryPortableMismatch::ResearchVersionIdentity => {
"hub.package.research_version_mismatch"
}
},
"portable discovery reference mismatch",
),
DiscoveryPortableError::Participant { .. } => validation(
"hub.package.invalid_participant",
"portable project participant is invalid",
),
DiscoveryPortableError::Portable(error) => validation(
portable_code(error.code),
"portable project verification failed",
),
}
}
#[cfg(test)]
mod core_version_tests;
#[cfg(test)]
mod download_digest_tests;
#[cfg(test)]
mod listing_tests;
#[cfg(test)]
mod research_clone_tests;
#[cfg(test)]
mod status_tests;
#[cfg(test)]
mod transfer_tests;
#[cfg(test)]
mod tests;
#[cfg(test)]
mod auth_tests;