use crate::hub_http::{
MAX_METADATA_BYTES, MAX_REDIRECTS, Transport, endpoint, network, parse_input,
validate_upload_url, validate_url, validation,
};
use clap::{ArgGroup, Args};
use graphforge_api::{
BuildResearchLineageRequest, CancellationToken, ExportResearchRequest, GfError, GraphForge,
PortableLimits, PortableSelection, PortableSelectionPreviewRequest, PortableSelectionProfile,
PortableSelectionRequest, ProjectErrorCode, ResearchReferenceTarget,
};
use graphforge_discovery::{
DiscoveryLimits, DiscoveryManifest, ObjectDescriptor, RESEARCH_LINEAGE_FORMAT,
RESEARCH_LINEAGE_MEDIA_TYPE, RefSet, RepositoryIdentity, ResearchLineage,
ResearchLineageReference, Sha256Digest,
};
use graphforge_hub_publish::{
CommitRequest, DEVICE_CODE_GRANT_TYPE, DeviceAuthorizationResponse, DevicePoll,
HUB_PUBLISH_FORMAT, HubErrorBody, HubMethod, HubPublishErrorCode, HubRequest, HubResponse,
ObjectDeclaration, OpenSessionRequest, OperationStatus, PUBLISH_NEGOTIATION_CAPABILITY,
PublishCapabilities, PublishIntent, PublishReceipt, PublishToken, SessionResponse,
UPLOAD_OFFSET_HEADER, UploadStatus, UploadTarget, canonical_json, content_range,
parse_unique_json, publish_scope, sha256_digest,
};
use publication::{
ManifestInputs, PACKAGE_MEDIA_TYPE, SUMMARY_MEDIA_TYPE, VerifiedBundle, build_manifest,
derive_summary, digest_bytes, export_module_packages, export_verified_bundle,
object_descriptor, summary_reference, verify_exported_bundle,
};
use serde_json::json;
use std::collections::{BTreeMap, BTreeSet};
use std::fs::File;
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use url::Url;
use uuid::Uuid;
mod history;
pub(crate) mod publication;
mod refusals;
static DEVICE_FLOW_ALLOWED: AtomicBool = AtomicBool::new(false);
pub(crate) fn allow_device_flow() {
DEVICE_FLOW_ALLOWED.store(true, Ordering::Relaxed);
}
const CLIENT_ID: &str = "graphforge-cli";
const MAX_CLIENT_CHUNK_BYTES: u64 = 1024 * 1024;
const MAX_NEGOTIATED_CHUNK_BYTES: u64 = 95 * 1024 * 1024;
const MAX_DEVICE_WAIT_SECONDS: u64 = 15 * 60;
const MAX_POLL_INTERVAL_SECONDS: u64 = 60;
const MAX_CONTROL_RESPONSE_BYTES: usize = 4 * 1024 * 1024;
const INTENT_FORMAT: &str = "graphforge-hub-publish-intent/1";
#[derive(Args)]
#[command(group(ArgGroup::new("selection").required(true).args(["git_ref", "version_uuid"])))]
pub(crate) struct PublishArgs {
#[arg(long)]
pub rewrite: bool,
#[arg(long)]
pub prune: bool,
#[arg(long)]
pub check_fork_origin: Option<String>,
pub repository: String,
#[arg(long = "ref")]
pub git_ref: Option<String>,
#[arg(long)]
pub version_uuid: Option<String>,
#[arg(long)]
pub fork_of: Option<String>,
#[arg(long)]
pub operation_uuid: Option<String>,
#[arg(long)]
pub staging_directory: Option<PathBuf>,
#[arg(long)]
pub protection_bypass_env: Option<String>,
#[arg(long = "preview")]
pub previews: Vec<std::path::PathBuf>,
}
pub(crate) enum Credential {
Token(PublishToken),
DeviceFlow,
}
pub(crate) struct PublishEnvironment<'a> {
pub bypass: Option<&'a crate::hub_http::DiscoveryCredentials>,
pub credential: Credential,
pub wait: &'a dyn Fn(Duration),
pub notices: &'a mut dyn Write,
resolution: Option<PublicationResolution>,
}
fn select_credential(
explicit: Option<PublishToken>,
streaming: bool,
interactive: bool,
saved: impl FnOnce() -> Result<Option<PublishToken>, GfError>,
) -> Result<Credential, GfError> {
if let Some(token) = explicit {
return Ok(Credential::Token(token));
}
if !streaming {
return Err(publish_error(
HubPublishErrorCode::AuthDenied,
"no publish credential: set GRAPHFORGE_HUB_PUBLISH_TOKEN or run gf publish in a terminal",
));
}
if let Some(token) = saved()? {
return Ok(Credential::Token(token));
}
if interactive {
return Ok(Credential::DeviceFlow);
}
Err(publish_error(
HubPublishErrorCode::AuthDenied,
"no saved session: run gf auth login or set GRAPHFORGE_HUB_PUBLISH_TOKEN",
))
}
#[derive(Debug, PartialEq)]
pub(crate) struct PublishOutcome {
pub receipt: PublishReceipt,
pub replayed: bool,
pub moved: Option<graphforge_discovery::RepositoryMoved>,
pub published: Option<PublishedHead>,
}
#[derive(Debug, PartialEq)]
pub(crate) struct PublishedHead {
pub branch_uuid: Uuid,
pub version_uuid: Uuid,
pub repository_url: String,
pub documents: Option<(DiscoveryManifest, ResearchLineage)>,
}
impl PublishedHead {
fn confirmed_by(
mut self,
receipt: &PublishReceipt,
manifest: &DiscoveryManifest,
lineage: &ResearchLineage,
) -> Self {
self.documents = manifest
.canonical_digest()
.is_ok_and(|digest| digest.0 == receipt.manifest_validator)
.then(|| (manifest.clone(), lineage.clone()));
self
}
}
pub(crate) fn run_hub_command(
command: crate::Command,
project: Option<PathBuf>,
project_dir: Option<PathBuf>,
json: bool,
output: &mut dyn Write,
diagnostics: &mut crate::Diagnostics<'_>,
) -> Result<i32, crate::CliRuntimeError> {
match command {
crate::Command::Clone(args) => {
Ok(crate::hub_clone::run_clone(args, json, output, diagnostics).map(|()| 0)?)
}
crate::Command::Publish(args) => {
run(&args, project, project_dir, json, output, diagnostics)
}
crate::Command::Hub(args) => {
Ok(crate::hub_clone::run_hub(&args, json, output).map(|()| 0)?)
}
_ => unreachable!("only Hub commands are dispatched here"),
}
}
fn run(
args: &PublishArgs,
project: Option<PathBuf>,
project_dir: Option<PathBuf>,
json: bool,
output: &mut dyn Write,
diagnostics: &mut crate::Diagnostics<'_>,
) -> Result<i32, crate::CliRuntimeError> {
run_command(args, project, project_dir, json, output, diagnostics)
.map(|()| 0)
.map_err(Into::into)
}
fn run_command(
args: &PublishArgs,
project: Option<PathBuf>,
project_dir: Option<PathBuf>,
json: bool,
output: &mut dyn Write,
diagnostics: &mut crate::Diagnostics<'_>,
) -> Result<(), GfError> {
run_command_with_token_env(args, None, project, project_dir, json, output, diagnostics)
}
pub(crate) fn run_command_with_token_env(
args: &PublishArgs,
token_env: Option<&str>,
project: Option<PathBuf>,
project_dir: Option<PathBuf>,
json: bool,
output: &mut dyn Write,
diagnostics: &mut crate::Diagnostics<'_>,
) -> Result<(), GfError> {
let (_, base) = parse_input(&args.repository)?;
if let Some(origin) = &args.fork_of {
parse_input(origin)?;
}
let bypass = crate::hub_http::DiscoveryCredentials::load(
&base,
None,
args.protection_bypass_env.as_deref(),
|name| std::env::var(name),
)?;
let explicit = publish_token(token_env)?;
let path = crate::resolve_project_path(project, project_dir)?;
let path = path
.to_str()
.ok_or_else(|| GfError::Validation("--project must be valid UTF-8".into()))?;
let transport = crate::hub_http::HubTransport::new();
publish_with_credentials(&transport, args, explicit, bypass.as_ref(), |environment| {
open_and_publish(
&transport,
path,
args,
environment,
json,
output,
diagnostics,
)
})
}
pub(crate) fn publish_token(token_env: Option<&str>) -> Result<Option<PublishToken>, GfError> {
let explicit = match token_env {
Some(name) => Some(PublishToken::new(std::env::var(name).map_err(|_| {
publish_error(
HubPublishErrorCode::AuthDenied,
"the --token-env variable is unset or not UTF-8",
)
})?)),
None => PublishToken::from_env(),
};
if explicit.is_none() && !DEVICE_FLOW_ALLOWED.load(Ordering::Relaxed) {
return Err(publish_error(
HubPublishErrorCode::AuthDenied,
"no publish credential: set GRAPHFORGE_HUB_PUBLISH_TOKEN or run gf publish in a terminal",
));
}
Ok(explicit)
}
pub(crate) fn publish_with_credentials(
transport: &dyn Transport,
args: &PublishArgs,
explicit: Option<PublishToken>,
bypass: Option<&crate::hub_http::DiscoveryCredentials>,
publish_with: impl FnOnce(&mut PublishEnvironment<'_>) -> Result<(), GfError>,
) -> Result<(), GfError> {
use std::io::IsTerminal as _;
let streaming = DEVICE_FLOW_ALLOWED.load(Ordering::Relaxed);
let interactive =
streaming && std::io::stdin().is_terminal() && std::io::stderr().is_terminal();
let (identity, base) = parse_input(&args.repository)?;
let resolution = resolve_publication(transport, &identity, &base, bypass)?;
let (identity, base) = (resolution.identity.clone(), resolution.base.clone());
let credential = select_credential(explicit, streaming, interactive, || {
let grant = crate::auth::repository_token(
&base,
&identity.owner,
&identity.repository,
"publish",
args.protection_bypass_env.as_deref(),
)?;
grant
.map(|grant| grant.into_publish_token(&base))
.transpose()
})?;
let stderr = std::io::stderr();
let mut notices = stderr.lock();
let mut environment = PublishEnvironment {
bypass,
credential,
wait: &std::thread::sleep,
notices: &mut notices,
resolution: Some(resolution),
};
publish_with(&mut environment)
}
fn open_and_publish(
transport: &dyn Transport,
path: &str,
args: &PublishArgs,
environment: &mut PublishEnvironment<'_>,
json: bool,
output: &mut dyn Write,
diagnostics: &mut crate::Diagnostics<'_>,
) -> Result<(), GfError> {
let mut graph = GraphForge::new(Some(path))?;
diagnostics.warn(&graph.open_warnings());
publish_and_report(transport, &mut graph, args, environment, json, output)
}
pub(crate) fn publish_and_report(
transport: &dyn Transport,
graph: &mut GraphForge,
args: &PublishArgs,
environment: &mut PublishEnvironment<'_>,
json: bool,
output: &mut dyn Write,
) -> Result<(), GfError> {
let outcome =
publish_with_operation_report(transport, graph, args, environment, |operation| {
if !json {
writeln!(output, "Operation UUID: {operation}")
.and_then(|()| output.flush())
.map_err(|error| GfError::Storage(error.to_string()))?;
}
Ok(())
})?;
record_upstream(graph, transport, args, &outcome)?;
write_outcome(&outcome, json, output)
}
fn record_upstream(
graph: &mut GraphForge,
transport: &dyn Transport,
args: &PublishArgs,
outcome: &PublishOutcome,
) -> Result<(), GfError> {
let Some(head) = &outcome.published else {
return Ok(());
};
let documents = match &head.documents {
Some(documents) => documents.clone(),
None if outcome.replayed => {
let tracked = graph
.research_version_retention()?
.remote_branches
.values()
.any(|citation| {
citation.branch_uuid == head.branch_uuid
&& citation.repository_url == head.repository_url
&& citation.head_version_uuid == head.version_uuid
});
if tracked {
return Ok(());
}
let served = crate::hub_clone::remote_read::read_remote(
transport,
&crate::hub_clone::remote_read::RemoteRequest {
repository: &head.repository_url,
hub: None,
token_env: None,
protection_bypass_env: args.protection_bypass_env.as_deref(),
},
None,
);
let Ok((served, _)) = served else {
return Ok(());
};
let Some(documents) = PublishedHead {
branch_uuid: head.branch_uuid,
version_uuid: head.version_uuid,
repository_url: head.repository_url.clone(),
documents: None,
}
.confirmed_by(&outcome.receipt, &served.manifest, &served.lineage)
.documents
else {
return Ok(());
};
documents
}
None => return Ok(()),
};
graph
.record_research_publication(
&graphforge_api::RecordResearchPublicationRequest {
operation_uuid: Uuid::now_v7(),
expected_generation_uuid: graph.committed_generation_identity()?.generation_uuid,
branch_uuid: head.branch_uuid,
repository_url: head.repository_url.clone(),
receipt: outcome.receipt.clone(),
manifest: documents.0,
lineage: documents.1,
},
&CancellationToken::new(),
)
.map(|_| ())
.map_err(|error| {
GfError::Validation(format!(
"the Hub confirmed the publication, but recording it as the Branch's upstream failed ({error}); rerun the same command to retry"
))
})
}
pub(crate) fn write_outcome(
outcome: &PublishOutcome,
json: bool,
output: &mut dyn Write,
) -> Result<(), GfError> {
let io = |error: std::io::Error| GfError::Storage(error.to_string());
if json {
let mut result = serde_json::to_value(&outcome.receipt)
.map_err(|error| GfError::Execution(error.to_string()))?;
if let Some(moved) = &outcome.moved {
result["moved_from"] = json!(moved.moved_from.canonical_name());
result["moved_to"] = json!(moved.moved_to.canonical_name());
}
serde_json::to_writer(&mut *output, &result)
.map_err(|error| GfError::Execution(error.to_string()))?;
writeln!(output).map_err(io)
} else {
if let Some(moved) = &outcome.moved {
writeln!(
output,
"Repository moved from {} to {}",
moved.moved_from.canonical_name(),
moved.moved_to.canonical_name()
)
.map_err(io)?;
}
writeln!(
output,
"Published {} ({}) at revision {}",
outcome.receipt.repository.canonical_name(),
outcome.receipt.refs.join(", "),
outcome.receipt.revision
)
.map_err(io)
}
}
struct PublicationResolution {
requested: RepositoryIdentity,
requested_base: Url,
identity: RepositoryIdentity,
base: Url,
moved: Option<graphforge_discovery::RepositoryMoved>,
capabilities: PublishCapabilities,
}
fn resolve_publication(
transport: &dyn Transport,
identity: &RepositoryIdentity,
base: &Url,
bypass: Option<&crate::hub_http::DiscoveryCredentials>,
) -> Result<PublicationResolution, GfError> {
let tracker = crate::hub_http::RepositoryMoveTransport::new(transport, identity, base)
.with_credentials(bypass);
let hub = Hub {
refusal: None,
transport: &tracker,
base: base.clone(),
bypass,
};
let (bytes, _) = hub
.read_control(&endpoint(base, "publish"), MAX_CONTROL_RESPONSE_BYTES)?
.ok_or_else(|| {
publish_error(
HubPublishErrorCode::InvalidInput,
"the Hub does not accept publications for this repository",
)
})?;
let (current, current_base, moved) = tracker.resolved(true)?;
let capabilities = PublishCapabilities::from_json(&bytes).map_err(|error| protocol(&error))?;
if capabilities.repository != current
|| capabilities.authorization.scope != publish_scope(¤t)
{
return Err(publish_error(
HubPublishErrorCode::InvalidInput,
"capabilities describe another repository",
));
}
Ok(PublicationResolution {
requested: identity.clone(),
requested_base: base.clone(),
identity: current,
base: current_base,
moved,
capabilities,
})
}
fn publish_error(code: HubPublishErrorCode, detail: &str) -> GfError {
let wire = graphforge_hub_publish::HubPublishError::new(code, "").wire_code();
match code {
HubPublishErrorCode::IdempotencyConflict => GfError::Project {
code: ProjectErrorCode::TransactionConflict,
message: format!("{wire}: {detail}"),
},
HubPublishErrorCode::Internal => GfError::Storage(format!("{wire}: {detail}")),
_ => GfError::Validation(format!("{wire}: {detail}")),
}
}
const fn code_detail(code: HubPublishErrorCode) -> &'static str {
match code {
HubPublishErrorCode::AuthDenied => "the Hub refused the publish credential",
HubPublishErrorCode::EntitlementDenied => "the Hub refused the publication entitlement",
HubPublishErrorCode::IdempotencyConflict => {
"the operation identity was already used for a different publication"
}
HubPublishErrorCode::RefConflict => {
"the repository changed since its revision was read; publish again"
}
HubPublishErrorCode::NonFastForward => {
"the proposed head does not descend from the published Branch"
}
HubPublishErrorCode::IdTaken => {
"the Hub refused a permanent item ID owned by another publisher"
}
HubPublishErrorCode::UnsupportedFuture => {
"the Hub requires an unsupported protocol version or capability"
}
HubPublishErrorCode::IntegrityFailure => {
"the Hub refused object bytes that disagree with their digest or length"
}
HubPublishErrorCode::InvalidInput => "the Hub refused the request as invalid",
HubPublishErrorCode::Internal => "the Hub failed internally",
}
}
fn hub_failure(response: &HubResponse) -> GfError {
match serde_json::from_slice::<HubErrorBody>(&response.body) {
Ok(body) if body.code == HubPublishErrorCode::IdTaken && response.status != 409 => {
network("Hub refusal status is invalid")
}
Ok(body) => publish_error(body.code, code_detail(body.code)),
Err(_) => network("Hub returned an unsuccessful status"),
}
}
fn protocol(error: &graphforge_hub_publish::HubPublishError) -> GfError {
publish_error(error.code, error.detail())
}
fn derivation(error: &str) -> GfError {
GfError::Validation(format!("hub.publish.derivation: {error}"))
}
fn project_package_profile(graph: &GraphForge) -> Result<PortableSelectionProfile, GfError> {
let plan = graph
.preview_portable_selection(&PortableSelectionPreviewRequest {
selection: PortableSelection::Current,
request: PortableSelectionRequest {
profile: PortableSelectionProfile::Complete,
strict: true,
},
limits: PortableLimits::default(),
})
.map_err(|error| derivation(&error.to_string()))?;
Ok(PortableSelectionProfile::Custom(
plan.included
.into_iter()
.filter(|entry| entry.kind != "research")
.map(|entry| entry.identity)
.collect(),
))
}
fn discovery(error: &graphforge_discovery::DiscoveryError) -> GfError {
let code = match error.code {
graphforge_discovery::DiscoveryErrorCode::UnsupportedFuture => {
HubPublishErrorCode::UnsupportedFuture
}
graphforge_discovery::DiscoveryErrorCode::IntegrityFailure => {
HubPublishErrorCode::IntegrityFailure
}
_ => HubPublishErrorCode::InvalidInput,
};
publish_error(code, error.detail())
}
type Document = (Vec<u8>, Option<String>);
struct Hub<'a> {
refusal: Option<&'a refusals::Capture>,
bypass: Option<&'a crate::hub_http::DiscoveryCredentials>,
transport: &'a dyn Transport,
base: Url,
}
impl Hub<'_> {
fn read(&self, url: &Url, limit: usize) -> Result<Option<Document>, GfError> {
self.read_inner(url, limit, false)
}
fn read_control(&self, url: &Url, limit: usize) -> Result<Option<Document>, GfError> {
self.read_inner(url, limit, true)
}
fn with_bypass(&self, mut request: HubRequest, url: &Url) -> Result<HubRequest, GfError> {
if url.origin() == self.base.origin()
&& let Some(bypass) = self.bypass
{
for (name, value) in bypass
.headers_for(url)?
.iter()
.filter(|(name, _)| *name == "x-vercel-protection-bypass")
{
request = request.with_header(
name,
value
.to_str()
.map_err(|_| validation("hub.auth_denied", "invalid preview credential"))?,
);
}
}
Ok(request)
}
fn read_inner(
&self,
url: &Url,
limit: usize,
control: bool,
) -> Result<Option<Document>, GfError> {
let mut url = url.clone();
for hop in 0..=MAX_REDIRECTS {
self.transport.validate(&url)?;
let request = HubRequest::new(HubMethod::Get, url.as_str());
let request = if control && hop == 0 {
self.with_bypass(request, &url)?
} else {
request
};
let response = self.transport.exchange(&request, limit)?;
match response.status {
301 | 302 | 303 | 307 | 308 if hop < MAX_REDIRECTS => {
let location = response
.header("location")
.ok_or_else(|| network("redirect is missing Location"))?;
url = url
.join(location)
.map_err(|_| validation("hub.unsafe_location", "invalid redirect URL"))?;
}
301 | 302 | 303 | 307 | 308 => return Err(network("redirect limit exceeded")),
404 => return Ok(None),
200..=299 => {
let etag = response.header("etag").map(str::to_owned);
return Ok(Some((response.body, etag)));
}
_ => return Err(hub_failure(&response)),
}
}
unreachable!("the redirect loop returns on its last hop")
}
fn control(&self, request: HubRequest, token: &PublishToken) -> Result<HubResponse, GfError> {
let url = Url::parse(&request.url)
.map_err(|_| validation("hub.unsafe_location", "control-plane URL is invalid"))?;
self.transport.validate(&url)?;
if url.origin() != self.base.origin() {
return Err(validation(
"hub.unsafe_location",
"control-plane URL is not on the repository origin",
));
}
let request = self.with_bypass(
request.with_header("authorization", token.authorization()),
&url,
)?;
let response = self
.transport
.exchange(&request, MAX_CONTROL_RESPONSE_BYTES)?;
if let Some(capture) = self.refusal {
capture.observe(
&response,
token,
request.header("x-vercel-protection-bypass"),
)?;
}
Ok(response)
}
fn data(&self, request: &HubRequest) -> Result<HubResponse, GfError> {
let url = Url::parse(&request.url)
.map_err(|_| validation("hub.unsafe_location", "upload URL is invalid"))?;
validate_upload_url(&url)?;
if request.headers.iter().any(|(name, _)| {
name.eq_ignore_ascii_case("authorization")
|| name.eq_ignore_ascii_case("x-vercel-protection-bypass")
}) {
return Err(validation(
"hub.auth_denied",
"upload capabilities cannot carry Hub credentials",
));
}
self.transport.exchange(request, MAX_CONTROL_RESPONSE_BYTES)
}
fn oauth(&self, endpoint: &str, form: &[(&str, &str)]) -> Result<HubResponse, GfError> {
let url = Url::parse(endpoint)
.map_err(|_| validation("hub.unsafe_location", "OAuth endpoint is invalid"))?;
self.transport.validate(&url)?;
let body = url::form_urlencoded::Serializer::new(String::new())
.extend_pairs(form)
.finish();
let request = HubRequest::new(HubMethod::Post, endpoint)
.with_header("content-type", "application/x-www-form-urlencoded")
.with_body(body.into_bytes());
let request = self.with_bypass(request, &url)?;
self.transport
.exchange(&request, MAX_CONTROL_RESPONSE_BYTES)
}
}
fn obtain_token(
hub: &Hub<'_>,
capabilities: &PublishCapabilities,
environment: &mut PublishEnvironment<'_>,
) -> Result<PublishToken, GfError> {
if let Credential::Token(token) = &environment.credential {
return Ok(token.clone());
}
let authorization = &capabilities.authorization;
let response = hub.oauth(
&authorization.device_authorization_endpoint,
&[
(
"client_id",
authorization.client_id.as_deref().unwrap_or(CLIENT_ID),
),
("scope", &authorization.scope),
],
)?;
if response.status != 200 {
return Err(publish_error(
HubPublishErrorCode::AuthDenied,
"device authorization was refused",
));
}
let device: DeviceAuthorizationResponse =
serde_json::from_slice(&response.body).map_err(|_| {
publish_error(
HubPublishErrorCode::InvalidInput,
"device authorization response is malformed",
)
})?;
let display_safe = |text: &str| {
!text.is_empty() && text.len() <= 512 && !text.bytes().any(|byte| byte.is_ascii_control())
};
if !display_safe(&device.user_code) || !display_safe(&device.verification_uri) {
return Err(publish_error(
HubPublishErrorCode::InvalidInput,
"device authorization response is malformed",
));
}
let io = |error: std::io::Error| GfError::Storage(error.to_string());
writeln!(
environment.notices,
"To publish {}, open {} and enter code {}",
capabilities.repository.canonical_name(),
device.verification_uri,
device.user_code
)
.map_err(io)?;
environment.notices.flush().map_err(io)?;
let mut interval = device.interval.clamp(1, MAX_POLL_INTERVAL_SECONDS);
let deadline = device.expires_in.min(MAX_DEVICE_WAIT_SECONDS);
let mut elapsed = 0_u64;
loop {
if elapsed >= deadline {
return Err(publish_error(
HubPublishErrorCode::AuthDenied,
if deadline < device.expires_in {
"device authorization did not complete within fifteen minutes"
} else {
"device authorization expired before approval"
},
));
}
let pause = interval.min(deadline - elapsed);
(environment.wait)(Duration::from_secs(pause));
elapsed = elapsed.saturating_add(pause);
let response = hub.oauth(
&authorization.token_endpoint,
&[
("grant_type", DEVICE_CODE_GRANT_TYPE),
("device_code", &device.device_code),
(
"client_id",
authorization.client_id.as_deref().unwrap_or(CLIENT_ID),
),
],
)?;
match DevicePoll::from_response(response.status, &response.body)
.map_err(|error| protocol(&error))?
{
DevicePoll::Granted(token) => return Ok(token),
DevicePoll::Pending => {}
DevicePoll::SlowDown => {
interval = interval.saturating_add(5).min(MAX_POLL_INTERVAL_SECONDS);
}
}
}
}
struct Selection {
git_ref: Option<String>,
branch_uuid: Option<Uuid>,
version_uuid: Uuid,
identity: Sha256Digest,
project_uuid: Uuid,
project_content: String,
publication_content: String,
}
fn hex_digest(bytes: &[u8; 32]) -> Sha256Digest {
Sha256Digest(
bytes
.iter()
.fold(String::from("sha256:"), |mut output, byte| {
use std::fmt::Write as _;
write!(output, "{byte:02x}").expect("writing to a string cannot fail");
output
}),
)
}
fn select(graph: &GraphForge, args: &PublishArgs) -> Result<Selection, GfError> {
let invalid = |detail: &str| publish_error(HubPublishErrorCode::InvalidInput, detail);
let registry = graph.research_version_retention()?;
let (branch_uuid, version_uuid) = match (&args.git_ref, &args.version_uuid) {
(Some(name), None) => {
check_ref_name(name)?;
let mut named = registry
.branches
.iter()
.filter(|(_, record)| record.label == *name);
let (branch_uuid, _) = named
.next()
.ok_or_else(|| invalid("no research Branch is labelled with this ref"))?;
if named.next().is_some() {
return Err(invalid("more than one research Branch has this label"));
}
let head = registry
.heads
.get(branch_uuid)
.ok_or_else(|| invalid("the research Branch has no head Version"))?;
(Some(*branch_uuid), *head)
}
(None, Some(version)) => (
None,
Uuid::parse_str(version).map_err(|_| invalid("version UUID is invalid"))?,
),
_ => return Err(invalid("specify exactly one of --ref and --version-uuid")),
};
let identity = registry
.identities
.get(&version_uuid)
.filter(|_| registry.versions.contains_key(&version_uuid))
.ok_or_else(|| invalid("the research Version is not retained in this Project"))?;
let reference = graph.research_reference(
&ResearchReferenceTarget::Version { version_uuid },
&CancellationToken::new(),
)?;
Ok(Selection {
git_ref: args.git_ref.clone(),
branch_uuid,
version_uuid,
identity: hex_digest(identity),
project_uuid: reference.project_uuid,
project_content: project_content(graph)?,
publication_content: hex_digest(&graph.publication_content_identity()?).0,
})
}
fn project_content(graph: &GraphForge) -> Result<String, GfError> {
Ok(hex_digest(&graph.committed_generation_identity()?.manifest_sha256).0)
}
fn check_ref_name(name: &str) -> Result<(), GfError> {
let probe = RefSet {
format: graphforge_discovery::DISCOVERY_FORMAT.into(),
version: graphforge_discovery::ProtocolVersion::CURRENT,
repository: RepositoryIdentity {
owner: "probe".into(),
repository: "probe".into(),
},
default_ref: name.into(),
refs: vec![graphforge_discovery::RepositoryRef {
name: name.into(),
target: Sha256Digest(sha256_digest(b"")),
validator: Sha256Digest(sha256_digest(b"")),
}],
extensions: BTreeMap::new(),
};
probe.validate(DiscoveryLimits::default()).map_err(|_| {
publish_error(
HubPublishErrorCode::InvalidInput,
"ref name is not a valid discovery ref",
)
})
}
fn intent_digest(
repository: &RepositoryIdentity,
selection: &Selection,
fork_of: Option<&RepositoryIdentity>,
) -> String {
sha256_digest(&canonical_json(&json!({
"format": INTENT_FORMAT,
"repository": repository.canonical_name(),
"ref": selection.git_ref,
"version_uuid": selection.version_uuid.to_string(),
"version_identity": selection.identity.0,
"project_uuid": selection.project_uuid.to_string(),
"project_content": selection.publication_content,
"fork_of": fork_of.map(RepositoryIdentity::canonical_name),
})))
}
struct SelectedPreview {
descriptor: graphforge_hub_publish::preview::PreviewDescriptor,
bytes: Vec<u8>,
}
fn select_previews(
args: &PublishArgs,
version_uuid: Uuid,
) -> Result<Vec<SelectedPreview>, GfError> {
use graphforge_hub_publish::preview as p;
use std::io::Read;
if args.previews.len() > p::MAX_PREVIEWS {
return Err(derivation("preview.limit"));
}
let mut selected = Vec::new();
let mut total = 0_u64;
for path in &args.previews {
let file = std::fs::File::open(path).map_err(|_| derivation("preview.content"))?;
let mut bytes = Vec::new();
file.take(p::MAX_PREVIEW_BYTES as u64 + 1)
.read_to_end(&mut bytes)
.map_err(|_| derivation("preview.content"))?;
total += bytes.len() as u64;
if total > p::MAX_TOTAL_PREVIEW_BYTES {
return Err(derivation("preview.limit"));
}
let media_type = if bytes.starts_with(b"\x89PNG\r\n\x1a\n") {
"image/png"
} else {
"image/svg+xml"
};
let (width, height) = p::validate_image(media_type, &bytes)
.map_err(|error| derivation(&error.to_string()))?;
selected.push(SelectedPreview {
descriptor: p::PreviewDescriptor {
version_uuid: version_uuid.to_string(),
digest: p::digest(&bytes),
media_type: media_type.into(),
length: bytes.len() as u64,
width,
height,
caption: None,
alt: None,
},
bytes,
});
}
p::validate_descriptors(
&selected
.iter()
.map(|p| p.descriptor.clone())
.collect::<Vec<_>>(),
)
.map_err(|error| derivation(&error.to_string()))?;
Ok(selected)
}
struct Published {
refs: RefSet,
revision: String,
manifest: DiscoveryManifest,
lineage: ResearchLineage,
}
fn read_published(
hub: &Hub<'_>,
base: &Url,
identity: &RepositoryIdentity,
) -> Result<Option<Published>, GfError> {
let tracker = crate::hub_http::RepositoryMoveTransport::new(hub.transport, identity, base)
.with_credentials(hub.bypass.filter(|_| base.origin() == hub.base.origin()));
let read_hub = Hub {
refusal: None,
transport: &tracker,
base: hub.base.clone(),
bypass: hub.bypass,
};
let Some((refs_bytes, etag)) =
read_hub.read_control(&endpoint(base, "refs"), MAX_METADATA_BYTES)?
else {
return Ok(None);
};
let (manifest_bytes, _) = read_hub
.read_control(&endpoint(base, "manifest"), MAX_METADATA_BYTES)?
.ok_or_else(|| {
publish_error(
HubPublishErrorCode::InvalidInput,
"repository refs exist without a manifest",
)
})?;
let (identity, _, _) = tracker.resolved(false)?;
bind_published(
&refs_bytes,
etag.as_deref(),
&manifest_bytes,
&identity,
|object, limit| read_object(hub, object, limit),
)
.map(Some)
}
fn read_private_context(
hub: &Hub<'_>,
identity: &RepositoryIdentity,
token: &PublishToken,
) -> Result<Option<Published>, GfError> {
let url = endpoint(&hub.base, "publish/context");
let response = hub.control(HubRequest::new(HubMethod::Get, url.as_str()), token)?;
if response.status != 200 {
return Err(hub_failure(&response));
}
if response.body.len() > MAX_CONTROL_RESPONSE_BYTES {
return Err(network("publish context exceeds response limit"));
}
let value = parse_unique_json(&response.body).map_err(|error| protocol(&error))?;
let malformed = || {
publish_error(
HubPublishErrorCode::InvalidInput,
"publish context must contain exactly refs_json, manifest_json and lineage_json, all strings or all null",
)
};
let object = value.as_object().ok_or_else(malformed)?;
let keys = ["refs_json", "manifest_json", "lineage_json"];
if object.len() != keys.len() || keys.iter().any(|key| !object.contains_key(*key)) {
return Err(malformed());
}
if keys.iter().all(|key| object[*key].is_null()) {
return Ok(None);
}
let refs = object["refs_json"].as_str().ok_or_else(malformed)?;
let manifest = object["manifest_json"].as_str().ok_or_else(malformed)?;
let lineage = object["lineage_json"].as_str().ok_or_else(malformed)?;
bind_published(
refs.as_bytes(),
None,
manifest.as_bytes(),
identity,
|object, limit| {
let bytes = lineage.as_bytes();
if bytes.len() > limit {
return Err(network("lineage exceeds response limit"));
}
validate_object_bytes(object, bytes)?;
Ok(bytes.to_vec())
},
)
.map(Some)
}
fn bind_published(
refs_bytes: &[u8],
etag: Option<&str>,
manifest_bytes: &[u8],
identity: &RepositoryIdentity,
load_lineage: impl FnOnce(&ObjectDescriptor, usize) -> Result<Vec<u8>, GfError>,
) -> Result<Published, GfError> {
let limits = DiscoveryLimits {
max_response_bytes: MAX_METADATA_BYTES,
..DiscoveryLimits::default()
};
let revision = sha256_digest(refs_bytes);
if etag.is_some_and(|etag| etag.trim_matches('"') != revision) {
return Err(publish_error(
HubPublishErrorCode::IntegrityFailure,
"refs validator does not match the refs document",
));
}
let refs = RefSet::from_json(refs_bytes, limits).map_err(|error| discovery(&error))?;
let manifest =
DiscoveryManifest::from_json(manifest_bytes, limits).map_err(|error| discovery(&error))?;
if refs.repository != *identity || manifest.repository != *identity {
return Err(publish_error(
HubPublishErrorCode::IntegrityFailure,
"discovery documents name another repository",
));
}
refs.validate_manifest(&manifest)
.map_err(|error| discovery(&error))?;
let validator = manifest
.canonical_digest()
.map_err(|error| discovery(&error))?;
if refs
.refs
.iter()
.find(|reference| reference.name == manifest.resolved_ref)
.is_none_or(|reference| reference.validator != validator)
{
return Err(publish_error(
HubPublishErrorCode::IntegrityFailure,
"ref validator does not match the canonical manifest",
));
}
let object = manifest.lineage_object().map_err(|_| {
publish_error(
HubPublishErrorCode::InvalidInput,
"the repository has no research lineage to extend",
)
})?;
let lineage_bytes = load_lineage(object, limits.max_lineage_bytes)?;
let lineage =
ResearchLineage::from_json(&lineage_bytes, limits).map_err(|error| discovery(&error))?;
manifest
.bind_lineage(&refs, &lineage)
.map_err(|error| discovery(&error))?;
Ok(Published {
refs,
revision,
manifest,
lineage,
})
}
fn read_object(hub: &Hub<'_>, object: &ObjectDescriptor, limit: usize) -> Result<Vec<u8>, GfError> {
let location = object.locations.first().ok_or_else(|| {
publish_error(HubPublishErrorCode::InvalidInput, "object has no location")
})?;
let url = Url::parse(location)
.map_err(|_| validation("hub.unsafe_location", "object location is invalid"))?;
validate_url(&url)?;
let (bytes, _) = hub.read(&url, limit)?.ok_or_else(|| {
publish_error(HubPublishErrorCode::InvalidInput, "object is not published")
})?;
validate_object_bytes(object, &bytes)?;
Ok(bytes)
}
fn validate_object_bytes(object: &ObjectDescriptor, bytes: &[u8]) -> Result<(), GfError> {
if digest_bytes(bytes) != object.digest.0 || bytes.len() as u64 != object.length {
return Err(publish_error(
HubPublishErrorCode::IntegrityFailure,
"object bytes disagree with their descriptor",
));
}
Ok(())
}
fn verify_fork_origin(
hub: &Hub<'_>,
origin_base: &Url,
fork: &graphforge_discovery::LineageForkOrigin,
) -> Result<(), GfError> {
let refused = || {
publish_error(
HubPublishErrorCode::IntegrityFailure,
"the origin repository does not publish the cited origin Version with its identity",
)
};
let origin = read_published(hub, origin_base, &fork.origin_repository)
.map_err(|_| refused())?
.ok_or_else(refused)?;
let version = origin
.lineage
.version(&fork.origin_version_uuid)
.ok_or_else(refused)?;
if origin.lineage.project_uuid != fork.origin_project_uuid
|| version.identity_digest != fork.origin_version_identity
{
return Err(refused());
}
Ok(())
}
enum Source {
File(PathBuf),
Bytes(Vec<u8>),
Carried,
}
struct Snapshot {
manifest: DiscoveryManifest,
refs: Vec<String>,
objects: Vec<ObjectDeclaration>,
sources: BTreeMap<String, Source>,
}
fn adopt_verified_credit(
published: &mut graphforge_discovery::LineageVersion,
local: &graphforge_discovery::LineageVersion,
) -> Result<(), GfError> {
for (old, new) in [
(&mut published.author, &local.author),
(&mut published.committer, &local.committer),
] {
if old.is_none() {
old.clone_from(new);
} else if old != new {
return Err(publish_error(
HubPublishErrorCode::IntegrityFailure,
"a published Version's credit differs from its identity-verified local record",
));
}
}
Ok(())
}
fn prune_history(next: &mut ResearchLineage) {
graphforge_hub_publish::prune_unreferenced_history(next);
let kept: BTreeSet<_> = next
.versions
.iter()
.map(|version| version.version_uuid.clone())
.collect();
next.tags.retain(|tag| kept.contains(&tag.version_uuid));
}
fn merge_tags(
previous: &ResearchLineage,
next: &mut ResearchLineage,
rewrite: bool,
) -> Result<(), GfError> {
let mut tags: BTreeMap<String, _> = previous
.tags
.iter()
.map(|tag| (tag.name.clone(), tag.clone()))
.collect();
for tag in next.tags.drain(..) {
if !rewrite
&& tags
.get(&tag.name)
.is_some_and(|served| served.version_uuid != tag.version_uuid)
{
return Err(GfError::Validation(format!(
"GF_HUB_TAG_CONFLICT: the repository already serves tag '{}' on another Version; publish with --rewrite to move it",
tag.name
)));
}
tags.insert(tag.name.clone(), tag);
}
next.tags = tags.into_values().collect();
Ok(())
}
fn merge_lineage(
previous: Option<&ResearchLineage>,
mut next: ResearchLineage,
rewrite: bool,
prune: bool,
) -> Result<ResearchLineage, GfError> {
let Some(previous) = previous else {
return Ok(next);
};
let invalid = |detail: &str| publish_error(HubPublishErrorCode::InvalidInput, detail);
if previous.project_uuid != next.project_uuid {
return Err(invalid(
"the repository publishes a different research Project",
));
}
if let Some(published) = &previous.fork {
next.fork = Some(
next.fork
.as_ref()
.and_then(|proposed| published.completed_by(proposed))
.ok_or_else(|| invalid("the repository cites a different Fork origin"))?,
);
}
let mut versions: BTreeMap<String, _> = previous
.versions
.iter()
.map(|version| (version.version_uuid.clone(), version.clone()))
.collect();
for version in next.versions.drain(..) {
match versions.get_mut(&version.version_uuid) {
Some(published) if published.identity_digest != version.identity_digest => {
return Err(publish_error(
HubPublishErrorCode::IntegrityFailure,
"a published Version has a different identity than the local Version",
));
}
Some(published) => {
adopt_verified_credit(published, &version)?;
if published.package.is_none() {
published.package = version.package;
}
}
None => {
versions.insert(version.version_uuid.clone(), version);
}
}
}
let mut branches: BTreeMap<String, _> = previous
.branches
.iter()
.map(|branch| (branch.branch_uuid.clone(), branch.clone()))
.collect();
for branch in next.branches.drain(..) {
if branches.values().any(|published| {
(published.ref_name == branch.ref_name) != (published.branch_uuid == branch.branch_uuid)
}) {
return Err(invalid(
"the ref already names another Branch, or the Branch is published under another ref",
));
}
branches.insert(branch.branch_uuid.clone(), branch);
}
let mut anchors: BTreeMap<_, _> = previous
.boundary_anchors
.iter()
.chain(&next.boundary_anchors)
.map(|anchor| (anchor.version_uuid.clone(), anchor.clone()))
.collect();
for anchor in &previous.boundary_anchors {
if next
.boundary_anchors
.iter()
.any(|new| new.version_uuid == anchor.version_uuid && new != anchor)
{
return Err(publish_error(
HubPublishErrorCode::IntegrityFailure,
"native Project-origin anchor changed",
));
}
}
anchors.retain(|id, _| !versions.contains_key(id));
next.boundary_anchors = anchors.into_values().collect();
next.versions = versions.into_values().collect();
next.branches = branches.into_values().collect();
next.proposals.clone_from(&previous.proposals);
merge_tags(previous, &mut next, rewrite)?;
let refs = next
.branches
.iter()
.map(|branch| branch.ref_name.clone())
.collect::<BTreeSet<_>>()
.into_iter()
.collect::<Vec<_>>();
if prune {
prune_history(&mut next);
}
graphforge_hub_publish::validate_history_transition(
previous,
&next,
&refs,
rewrite,
prune,
DiscoveryLimits::default(),
)
.map_err(|error| protocol(&error))?;
Ok(next)
}
#[allow(clippy::too_many_lines)]
fn build_snapshot(
graph: &GraphForge,
identity: &RepositoryIdentity,
capabilities: &PublishCapabilities,
selection: &Selection,
history: history::SnapshotHistory<'_>,
scratch: &Path,
) -> Result<(Snapshot, ResearchLineage), GfError> {
let published = history.published;
let fork_origin_repository = history.fork_origin_repository;
let cancellation = CancellationToken::new();
let location = |digest: &str| capabilities.object_location(digest);
let mut sources = BTreeMap::new();
let mut objects = Vec::new();
let carried_version = published.and_then(|published| {
published
.lineage
.version(&selection.version_uuid.to_string())
.and_then(|version| version.package.clone())
});
let version_package = if let Some(package) = carried_version {
package
} else {
let path = scratch.join(format!("version-{}.gfpb", selection.version_uuid));
let receipt = graph
.export_research(
&ExportResearchRequest {
version_uuid: selection.version_uuid,
output: path.clone(),
bundled: true,
projection: None,
},
&cancellation,
)
.map_err(|error| GfError::Validation(format!("hub.publish.derivation: {error}")))?;
let bundle =
verify_exported_bundle(&path, &receipt.package_digest, &receipt.transport_digest)
.map_err(|error| derivation(&error))?;
add_bundle(&mut objects, &mut sources, &bundle, &location);
bundle.package_reference()
};
let project = export_verified_bundle(
graph,
project_package_profile(graph)?,
&scratch.join("project.gfpb"),
)
.map_err(|error| derivation(&error))?;
add_bundle(&mut objects, &mut sources, &project, &location);
let immutable_version = Sha256Digest(project.package_digest.clone());
let lineage = graph.build_research_lineage_for_discovery(
&BuildResearchLineageRequest {
repository: identity.clone(),
immutable_version: immutable_version.clone(),
project_uuid: selection.project_uuid,
branch_ref_names: selection
.branch_uuid
.zip(selection.git_ref.clone())
.into_iter()
.collect(),
version_packages: BTreeMap::from([(selection.version_uuid, version_package)]),
fork_origin_repository,
published_proposals: BTreeSet::new(),
},
&cancellation,
)?;
let lineage = merge_lineage(
published.map(|published| &published.lineage),
lineage,
history.rewrite,
history.prune,
)?;
let summary = derive_summary(identity, &immutable_version, &project.path, Some(&lineage))
.map_err(|error| derivation(&error))?;
let summary_object = object_descriptor(
digest_bytes(&summary.bytes),
summary.bytes.len() as u64,
SUMMARY_MEDIA_TYPE,
location(&digest_bytes(&summary.bytes)),
);
sources.insert(
summary_object.digest.0.clone(),
Source::Bytes(summary.bytes.clone()),
);
objects.push(summary_object.clone());
let (ontology, modules) = export_module_packages(graph, &summary.summary, |digest| {
Ok(scratch.join(format!(
"module-{}.gfpb",
digest.trim_start_matches("sha256:")
)))
})
.map_err(|error| derivation(&error))?;
for module in &modules {
add_bundle(&mut objects, &mut sources, &module.bundle, &location);
}
let carried: BTreeSet<&Sha256Digest> = lineage
.versions
.iter()
.filter_map(|version| version.package.as_ref())
.chain(lineage.proposals.iter().map(|proposal| &proposal.package))
.map(|package| &package.object_digest)
.collect();
for object_digest in carried {
if sources.contains_key(&object_digest.0) {
continue;
}
let previous = published
.and_then(|published| {
published
.manifest
.objects
.iter()
.find(|object| object.digest == *object_digest)
})
.ok_or_else(|| {
publish_error(
HubPublishErrorCode::IntegrityFailure,
"a published package object is absent from the repository manifest",
)
})?;
objects.push(object_descriptor(
previous.digest.0.clone(),
previous.length,
&previous.media_type,
location(&previous.digest.0),
));
sources.insert(previous.digest.0.clone(), Source::Carried);
}
let lineage_bytes = lineage
.to_canonical_json()
.map_err(|error| discovery(&error))?;
let lineage_digest = lineage
.canonical_digest()
.map_err(|error| discovery(&error))?;
let lineage_object = object_descriptor(
digest_bytes(&lineage_bytes),
lineage_bytes.len() as u64,
RESEARCH_LINEAGE_MEDIA_TYPE,
location(&digest_bytes(&lineage_bytes)),
);
sources.insert(
lineage_object.digest.0.clone(),
Source::Bytes(lineage_bytes),
);
objects.push(lineage_object.clone());
let default_ref = match (published, &selection.git_ref) {
(Some(published), _) => published.refs.default_ref.clone(),
(None, Some(name)) => name.clone(),
(None, None) => {
return Err(publish_error(
HubPublishErrorCode::InvalidInput,
"creating a repository needs --ref to name its default Branch",
));
}
};
let resolved_ref = selection
.git_ref
.clone()
.unwrap_or_else(|| default_ref.clone());
let manifest = build_manifest(ManifestInputs {
repository: identity.clone(),
default_ref: default_ref.clone(),
resolved_ref: resolved_ref.clone(),
package: project.package_reference(),
summary: Some(summary_reference(&summary, &summary_object)),
ontology,
lineage: Some(ResearchLineageReference {
format: RESEARCH_LINEAGE_FORMAT.into(),
lineage_digest,
object_digest: lineage_object.digest.clone(),
}),
objects,
})
.map_err(|error| derivation(&error))?;
manifest
.bind_summary(&summary.summary)
.map_err(|error| discovery(&error))?;
let refs: BTreeSet<String> = lineage
.branches
.iter()
.map(|branch| branch.ref_name.clone())
.chain([default_ref, resolved_ref])
.collect();
let declarations = manifest
.objects
.iter()
.map(|object| ObjectDeclaration {
digest: object.digest.0.clone(),
length: object.length,
media_type: object.media_type.clone(),
})
.collect();
Ok((
Snapshot {
manifest,
refs: refs.into_iter().collect(),
objects: declarations,
sources,
},
lineage,
))
}
fn add_bundle(
objects: &mut Vec<ObjectDescriptor>,
sources: &mut BTreeMap<String, Source>,
bundle: &VerifiedBundle,
location: &dyn Fn(&str) -> String,
) {
objects.push(object_descriptor(
bundle.object_digest.clone(),
bundle.length,
PACKAGE_MEDIA_TYPE,
location(&bundle.object_digest),
));
sources.insert(
bundle.object_digest.clone(),
Source::File(bundle.path.clone()),
);
}
fn read_chunk(source: &Source, offset: u64, length: u64) -> Result<Vec<u8>, GfError> {
let io = |error: std::io::Error| GfError::Storage(error.to_string());
match source {
Source::Bytes(bytes) => {
let start = usize::try_from(offset).map_err(|_| network("offset overflow"))?;
let end = usize::try_from(offset + length).map_err(|_| network("offset overflow"))?;
bytes
.get(start..end)
.map(<[u8]>::to_vec)
.ok_or_else(|| network("upload range is outside the object"))
}
Source::File(path) => {
let mut file = File::open(path).map_err(io)?;
file.seek(SeekFrom::Start(offset)).map_err(io)?;
let size = usize::try_from(length).map_err(|_| network("chunk overflow"))?;
let mut buffer = Vec::new();
buffer
.try_reserve_exact(size)
.map_err(|_| network("chunk allocation exceeds available memory"))?;
buffer.resize(size, 0);
file.read_exact(&mut buffer).map_err(io)?;
Ok(buffer)
}
Source::Carried => Err(publish_error(
HubPublishErrorCode::InvalidInput,
"the Hub has not admitted an object an earlier publication carried",
)),
}
}
fn upload_offset(response: &HubResponse, length: u64) -> Result<u64, GfError> {
response
.header(UPLOAD_OFFSET_HEADER)
.and_then(|value| value.trim().parse::<u64>().ok())
.filter(|offset| *offset <= length)
.ok_or_else(|| {
publish_error(
HubPublishErrorCode::InvalidInput,
"upload offset is missing or out of range",
)
})
}
fn negotiated_chunk_size(limits: &graphforge_hub_publish::PublishLimits) -> Result<u64, GfError> {
let size = limits
.preferred_chunk_bytes
.unwrap_or(limits.max_chunk_bytes.min(MAX_CLIENT_CHUNK_BYTES));
if size > MAX_NEGOTIATED_CHUNK_BYTES {
return Err(publish_error(
HubPublishErrorCode::InvalidInput,
"preferred upload chunk exceeds the client's 95 MiB buffer budget",
));
}
Ok(size)
}
fn upload_chunk_size(
capabilities: &PublishCapabilities,
target: &UploadTarget,
) -> Result<u64, GfError> {
let mut limits = capabilities.limits;
limits.preferred_chunk_bytes = capabilities
.upload_chunk_bytes(target)
.map_err(|error| protocol(&error))?;
negotiated_chunk_size(&limits)
}
fn upload(
hub: &Hub<'_>,
target: &UploadTarget,
source: &Source,
max_chunk: u64,
uniform: bool,
) -> Result<(), GfError> {
if target.received >= target.length {
return Ok(());
}
let head = hub.data(&HubRequest::new(HubMethod::Head, &target.upload_url))?;
if head.status != 200 {
return Err(hub_failure(&head));
}
let mut offset = upload_offset(&head, target.length)?;
let mut resynchronized = false;
while offset < target.length {
if uniform && offset % max_chunk != 0 {
return Err(publish_error(
HubPublishErrorCode::InvalidInput,
"retained offset is not aligned to a uniform upload part",
));
}
let chunk = max_chunk.min(target.length - offset);
let bytes = read_chunk(source, offset, chunk)?;
let response = hub.data(
&HubRequest::new(HubMethod::Put, &target.upload_url)
.with_header("content-type", "application/octet-stream")
.with_header("content-range", content_range(offset, chunk, target.length))
.with_body(bytes),
)?;
match response.status {
200 => {
let status: UploadStatus =
serde_json::from_slice(&response.body).map_err(|_| {
publish_error(
HubPublishErrorCode::InvalidInput,
"upload status is malformed",
)
})?;
if status.digest != target.digest
|| status.length != target.length
|| status.received != offset + chunk
{
return Err(publish_error(
HubPublishErrorCode::IntegrityFailure,
"upload status disagrees with the bytes sent",
));
}
offset = status.received;
resynchronized = false;
}
416 if !resynchronized => {
offset = upload_offset(&response, target.length)?;
resynchronized = true;
}
_ => return Err(hub_failure(&response)),
}
}
Ok(())
}
fn check_uniform_inventory(
capabilities: &PublishCapabilities,
uploads: &[UploadTarget],
chunks: &[u64],
) -> Result<(), GfError> {
if capabilities
.requirements
.iter()
.any(|c| c.capability == "large-object-publish" && c.major == 1)
{
for (target, size) in uploads.iter().zip(chunks) {
if target.length.div_ceil(*size) > 10_000
|| (target.received != target.length && target.received % size != 0)
{
return Err(publish_error(
HubPublishErrorCode::InvalidInput,
"object exceeds uniform multipart bounds or has an unaligned retained offset",
));
}
}
}
Ok(())
}
fn prepare_staging(
graph: &GraphForge,
explicit: Option<&Path>,
) -> Result<tempfile::TempDir, GfError> {
prepare_staging_with_space(graph, explicit, |path| fs4::available_space(path))
}
fn prepare_staging_with_space(
graph: &GraphForge,
explicit: Option<&Path>,
available: impl FnOnce(&Path) -> std::io::Result<u64>,
) -> Result<tempfile::TempDir, GfError> {
let parent = explicit
.or_else(|| {
graph.path().and_then(Path::parent).map(|parent| {
if parent.as_os_str().is_empty() {
Path::new(".")
} else {
parent
}
})
})
.ok_or_else(|| {
GfError::Validation(
"publication requires an explicit staging directory for an in-memory Project"
.into(),
)
})?;
if !parent.is_dir() {
return Err(GfError::Validation(
"publication staging directory must already exist".into(),
));
}
let needed = graph
.publication_payload_bytes()?
.checked_mul(3)
.and_then(|n| n.checked_add(512 * 1024 * 1024))
.ok_or_else(|| GfError::Storage("publication staging size overflow".into()))?;
if available(parent).map_err(|error| GfError::Storage(error.to_string()))? < needed {
return Err(GfError::Storage(
"insufficient free space for publication staging".into(),
));
}
let scratch = tempfile::Builder::new()
.prefix(".graphforge-publish-")
.tempdir_in(parent)
.map_err(|error| GfError::Storage(error.to_string()))?;
graphforge_api::filesystem_durability_preflight(scratch.path().join("admission"))?;
Ok(scratch)
}
fn poll_commit(
hub: &Hub<'_>,
base: &Url,
token: &PublishToken,
identity: &RepositoryIdentity,
operation_uuid: Uuid,
intent_digest: &str,
wait: &dyn Fn(Duration),
) -> Result<PublishReceipt, GfError> {
let url = endpoint(base, &format!("publish/operations/{operation_uuid}"));
let deadline = std::time::Instant::now() + Duration::from_hours(24);
while std::time::Instant::now() < deadline {
let response = hub.control(HubRequest::new(HubMethod::Get, url.as_str()), token)?;
if response.status != 200 {
return Err(hub_failure(&response));
}
let status =
OperationStatus::from_json(&response.body).map_err(|error| protocol(&error))?;
if status.repository != *identity
|| status.operation_uuid != operation_uuid
|| status.intent_digest != intent_digest
{
return Err(publish_error(
HubPublishErrorCode::IntegrityFailure,
"operation status does not bind this publication",
));
}
if let Some(receipt) = status.receipt {
return Ok(receipt);
}
wait(Duration::from_millis(100));
}
Err(publish_error(
HubPublishErrorCode::InvalidInput,
"asynchronous commit remains pending; rerun the same operation to resume",
))
}
fn session_response(response: &HubResponse) -> Result<SessionResponse, GfError> {
if !(200..300).contains(&response.status) {
return Err(hub_failure(response));
}
serde_json::from_slice(&response.body).map_err(|_| {
publish_error(
HubPublishErrorCode::InvalidInput,
"session response is malformed",
)
})
}
fn check_receipt(
receipt: &PublishReceipt,
identity: &RepositoryIdentity,
operation_uuid: Uuid,
) -> Result<(), GfError> {
if receipt.format != HUB_PUBLISH_FORMAT
|| &receipt.repository != identity
|| receipt.operation_uuid != operation_uuid
{
return Err(publish_error(
HubPublishErrorCode::IntegrityFailure,
"the receipt does not describe this operation",
));
}
Ok(())
}
fn check_ontology_composition(graph: &GraphForge, selection: &Selection) -> Result<(), GfError> {
let current = graph
.portable_ontology_composition(PortableLimits::default())
.map_err(|error| derivation(&error.to_string()))?;
let selected = graph
.research_version_portable_ontology_composition(
selection.version_uuid,
PortableLimits::default(),
)
.map_err(|error| derivation(&error.to_string()))?;
if current != selected {
return Err(validation(
"hub.publish.ontology_composition_mismatch",
&format!(
"workspace ontology composition differs from selected Version {}; check out that Version, or commit the ontology changes as a new Version and publish the current Branch head",
selection.version_uuid
),
));
}
Ok(())
}
#[cfg(test)]
pub(crate) fn publish(
transport: &dyn Transport,
graph: &GraphForge,
args: &PublishArgs,
environment: &mut PublishEnvironment<'_>,
) -> Result<PublishOutcome, GfError> {
publish_with_operation_report(transport, graph, args, environment, |_| Ok(()))
}
fn publish_with_operation_report(
transport: &dyn Transport,
graph: &GraphForge,
args: &PublishArgs,
environment: &mut PublishEnvironment<'_>,
report: impl FnMut(Uuid) -> Result<(), GfError>,
) -> Result<PublishOutcome, GfError> {
let captured = refusals::Capture::default();
let result = publish_inner(transport, graph, args, environment, &captured, report);
if result.is_err()
&& let Some((version, items)) = captured.take()
{
return Err(refusals::record(graph, version, &items));
}
result
}
#[allow(clippy::too_many_lines)]
fn publish_inner(
transport: &dyn Transport,
graph: &GraphForge,
args: &PublishArgs,
environment: &mut PublishEnvironment<'_>,
captured: &refusals::Capture,
mut report: impl FnMut(Uuid) -> Result<(), GfError>,
) -> Result<PublishOutcome, GfError> {
let (identity, base) = parse_input(&args.repository)?;
let fork = args.fork_of.as_deref().map(parse_input).transpose()?;
if fork.as_ref().is_some_and(|(origin, _)| *origin == identity) {
return Err(publish_error(
HubPublishErrorCode::InvalidInput,
"a Fork must publish a repository other than its origin",
));
}
let operation_uuid = args
.operation_uuid
.as_deref()
.map(Uuid::parse_str)
.transpose()
.map_err(|_| {
publish_error(
HubPublishErrorCode::InvalidInput,
"operation UUID is invalid",
)
})?;
let selection = select(graph, args)?;
check_ontology_composition(graph, &selection)?;
captured.select(selection.version_uuid);
let previews = select_previews(args, selection.version_uuid)?;
let preview_metadata = previews
.iter()
.map(|p| p.descriptor.clone())
.collect::<Vec<_>>();
let operation_content = if previews.is_empty() {
selection.publication_content.clone()
} else {
sha256_digest(&canonical_json(
&json!({"project_content":selection.publication_content,"previews":preview_metadata}),
))
};
let operation_content = history::bind_flags(operation_content, args);
let resolution = match environment.resolution.take() {
Some(resolution) => {
if resolution.requested != identity || resolution.requested_base != base {
return Err(validation(
"hub.integrity_failure",
"resolved publication does not match the request",
));
}
resolution
}
None => resolve_publication(transport, &identity, &base, environment.bypass)?,
};
let PublicationResolution {
identity,
base,
moved,
capabilities,
..
} = resolution;
let confirmed = selection
.branch_uuid
.map(|branch_uuid| PublishedHead {
branch_uuid,
version_uuid: selection.version_uuid,
repository_url: format!(
"{}/{}",
base.origin().ascii_serialization(),
identity.canonical_name()
),
documents: None,
})
.filter(|_| selection.git_ref.is_some());
let operation_uuid = operation_uuid.unwrap_or_else(|| {
graphforge_api::hub_publish_operation(
&identity.canonical_name(),
selection.git_ref.as_deref(),
&selection.version_uuid.to_string(),
&operation_content,
)
});
report(operation_uuid)?;
let base_intent = intent_digest(&identity, &selection, fork.as_ref().map(|(id, _)| id));
let intent_digest = if previews.is_empty() {
base_intent
} else {
sha256_digest(&canonical_json(
&json!({"intent":base_intent,"previews":preview_metadata}),
))
};
let intent_digest = history::bind_flags(intent_digest, args);
graphforge_hub_publish::require_history_capability(&capabilities, args.rewrite, args.prune)
.map_err(|error| protocol(&error))?;
let hub = Hub {
refusal: Some(captured),
bypass: environment.bypass,
transport,
base: base.clone(),
};
if !previews.is_empty()
&& !capabilities
.capabilities
.iter()
.any(|c| c.capability == "version-previews" && c.major == 1)
{
return Err(publish_error(
HubPublishErrorCode::UnsupportedFuture,
"Hub does not support the selected preview channel",
));
}
let token = obtain_token(&hub, &capabilities, environment)?;
let read_current = || {
if capabilities.capabilities.iter().any(|capability| {
capability.capability == "publish-read-context" && capability.major == 1
}) {
read_private_context(&hub, &identity, &token)
} else {
read_published(&hub, &base, &identity)
}
};
let status_url = endpoint(&base, &format!("publish/operations/{operation_uuid}"));
let status = hub.control(HubRequest::new(HubMethod::Get, status_url.as_str()), &token)?;
match status.status {
200 => {
let status =
OperationStatus::from_json(&status.body).map_err(|error| protocol(&error))?;
if status.repository != identity || status.operation_uuid != operation_uuid {
return Err(publish_error(
HubPublishErrorCode::IntegrityFailure,
"operation status describes another operation",
));
}
if status.intent_digest != intent_digest {
return Err(publish_error(
HubPublishErrorCode::IdempotencyConflict,
code_detail(HubPublishErrorCode::IdempotencyConflict),
));
}
if let Some(receipt) = status.receipt {
check_receipt(&receipt, &identity, operation_uuid)?;
if args.check_fork_origin.is_some() {
let published = read_current()?;
history::verify_established_fork(
&hub,
graph,
published.as_ref(),
selection.project_uuid,
args.check_fork_origin.as_deref(),
environment.notices,
)?;
}
return Ok(PublishOutcome {
receipt,
replayed: true,
moved,
published: confirmed,
});
}
}
404 => {}
_ => return Err(hub_failure(&status)),
}
let published = read_current()?;
let fork_origin_repository = match (&fork, &published) {
(Some(_), Some(_)) => {
return Err(publish_error(
HubPublishErrorCode::RefConflict,
"a Fork publishes a new repository, and this repository already exists",
));
}
(Some((origin, _)), None) => Some(origin.clone()),
(None, Some(published)) => published
.lineage
.fork
.as_ref()
.map(|fork| fork.origin_repository.clone()),
(None, None) => None,
};
let scratch = prepare_staging(graph, args.staging_directory.as_deref())?;
let (mut snapshot, lineage) = build_snapshot(
graph,
&identity,
&capabilities,
&selection,
history::SnapshotHistory {
published: published.as_ref(),
fork_origin_repository,
rewrite: args.rewrite,
prune: args.prune,
},
scratch.path(),
)?;
if !previews.is_empty() {
use graphforge_hub_publish::preview as p;
for item in previews {
snapshot.manifest.objects.push(object_descriptor(
item.descriptor.digest.clone(),
item.descriptor.length,
&item.descriptor.media_type,
capabilities.object_location(&item.descriptor.digest),
));
snapshot
.sources
.insert(item.descriptor.digest, Source::Bytes(item.bytes));
}
snapshot
.manifest
.objects
.sort_by(|a, b| a.digest.cmp(&b.digest));
snapshot.manifest.extensions.insert(
p::PREVIEW_EXTENSION.into(),
serde_json::to_value(p::PreviewSet {
format: "graphforge-previews/1".into(),
previews: preview_metadata,
})
.map_err(|_| derivation("preview.content"))?,
);
snapshot
.manifest
.validate(DiscoveryLimits::default())
.map_err(|error| discovery(&error))?;
snapshot.objects = snapshot
.manifest
.objects
.iter()
.map(|object| ObjectDeclaration {
digest: object.digest.0.clone(),
length: object.length,
media_type: object.media_type.clone(),
})
.collect();
}
if project_content(graph)? != selection.project_content {
return Err(publish_error(
HubPublishErrorCode::InvalidInput,
"the Project changed while it was being published; publish again",
));
}
if let Some((_, origin_base)) = &fork {
let cited = lineage.fork.as_ref().ok_or_else(|| {
publish_error(
HubPublishErrorCode::InvalidInput,
"this Project is not a Fork",
)
})?;
verify_fork_origin(&hub, origin_base, cited)?;
}
history::verify_established_fork(
&hub,
graph,
published.as_ref(),
selection.project_uuid,
args.check_fork_origin.as_deref(),
environment.notices,
)?;
if args.prune {
writeln!(
environment.notices,
"Pruning repository history; independent Fork copies and citations are retained."
)
.map_err(|error| GfError::Execution(error.to_string()))?;
}
let manifest_validator = snapshot
.manifest
.canonical_digest()
.map_err(|error| discovery(&error))?
.0;
let expected_revision = published
.as_ref()
.map(|published| published.revision.clone());
let request_commitment = PublishIntent {
rewrite: args.rewrite,
prune: args.prune,
repository: identity.clone(),
intent_digest: intent_digest.clone(),
objects: snapshot.objects.clone(),
manifest_validator: manifest_validator.clone(),
refs: snapshot.refs.clone(),
expected_revision: expected_revision.clone(),
}
.request_commitment();
let open = OpenSessionRequest {
format: HUB_PUBLISH_FORMAT.into(),
operation_uuid,
request_commitment: request_commitment.clone(),
intent_digest,
repository: identity.clone(),
objects: snapshot.objects.clone(),
requirements: capabilities
.requirements
.iter()
.filter(|capability| {
matches!(
capability.capability.as_str(),
PUBLISH_NEGOTIATION_CAPABILITY | "large-object-publish"
)
})
.cloned()
.chain(
(!args.previews.is_empty())
.then(|| graphforge_hub_publish::Capability::new("version-previews", 1)),
)
.chain((args.rewrite || args.prune).then(|| {
graphforge_hub_publish::Capability::new(
graphforge_hub_publish::HISTORY_MUTATION_CAPABILITY,
1,
)
}))
.collect(),
};
negotiated_chunk_size(&capabilities.limits)?;
let sessions = endpoint(&base, "publish/sessions");
let opened = session_response(&hub.control(
HubRequest::new(HubMethod::Post, sessions.as_str()).with_json(&open),
&token,
)?)?;
let (session_id, uploads) = match opened {
SessionResponse::Complete { receipt } => {
check_receipt(&receipt, &identity, operation_uuid)?;
let confirmed =
confirmed.map(|head| head.confirmed_by(&receipt, &snapshot.manifest, &lineage));
return Ok(PublishOutcome {
receipt,
replayed: true,
moved,
published: confirmed,
});
}
SessionResponse::Open {
session_id,
uploads,
} => (session_id, uploads),
SessionResponse::Pending { .. } => {
return Err(publish_error(
HubPublishErrorCode::InvalidInput,
"session open returned an unexpected pending state",
));
}
};
if uploads.len() != snapshot.objects.len()
|| uploads
.iter()
.zip(&snapshot.objects)
.any(|(target, object)| {
target.digest != object.digest || target.length != object.length
})
{
return Err(publish_error(
HubPublishErrorCode::InvalidInput,
"upload locations do not match the declared inventory",
));
}
let chunk_sizes = uploads
.iter()
.map(|target| upload_chunk_size(&capabilities, target))
.collect::<Result<Vec<_>, _>>()?;
check_uniform_inventory(&capabilities, &uploads, &chunk_sizes)?;
for (target, max_chunk) in uploads.iter().zip(chunk_sizes) {
let source = snapshot.sources.get(&target.digest).ok_or_else(|| {
publish_error(
HubPublishErrorCode::InvalidInput,
"the Hub issued an upload for an undeclared object",
)
})?;
upload(
&hub,
target,
source,
max_chunk,
capabilities
.requirements
.iter()
.any(|c| c.capability == "large-object-publish" && c.major == 1),
)?;
}
let commit_url = endpoint(&base, &format!("publish/sessions/{session_id}/commit"));
let manifest_value = serde_json::to_value(&snapshot.manifest)
.map_err(|error| GfError::Execution(error.to_string()))?;
let committed = session_response(&hub.control(
HubRequest::new(HubMethod::Post, commit_url.as_str()).with_json(&CommitRequest {
rewrite: args.rewrite,
prune: args.prune,
manifest: manifest_value,
refs: snapshot.refs.clone(),
expected_revision,
}),
&token,
)?)?;
let receipt = match committed {
SessionResponse::Complete { receipt } => receipt,
SessionResponse::Pending {
operation_uuid: pending,
} if pending == operation_uuid
&& capabilities
.requirements
.iter()
.any(|c| c.capability == "large-object-publish" && c.major == 1) =>
{
poll_commit(
&hub,
&base,
&token,
&identity,
operation_uuid,
&open.intent_digest,
environment.wait,
)?
}
_ => {
return Err(publish_error(
HubPublishErrorCode::InvalidInput,
"commit did not complete or accept this publication",
));
}
};
check_receipt(&receipt, &identity, operation_uuid)?;
if receipt.request_commitment != request_commitment
|| receipt.manifest_validator != manifest_validator
{
return Err(publish_error(
HubPublishErrorCode::IntegrityFailure,
"the receipt does not bind the committed publication",
));
}
let confirmed = confirmed.map(|head| head.confirmed_by(&receipt, &snapshot.manifest, &lineage));
Ok(PublishOutcome {
receipt,
replayed: false,
moved,
published: confirmed,
})
}
#[cfg(test)]
mod tests;
#[cfg(test)]
mod preview_tests;