use anyhow::Context;
use docker_credential::{CredentialRetrievalError, DockerCredential};
use futures_util::{stream, StreamExt, TryStreamExt};
use http::HeaderValue;
use oci_client::{
client::{ClientConfig, ClientProtocol},
errors::DigestError,
secrets::RegistryAuth,
Client, Reference, RegistryOperation,
};
use std::{env, io, num::NonZeroUsize};
use tokio::runtime::{Builder as RuntimeBuilder, Runtime};
const ARTIFACT_TRANSFER_CONCURRENCY_ENV: &str = "OMMX_ARTIFACT_TRANSFER_CONCURRENCY";
const DEFAULT_ARTIFACT_TRANSFER_CONCURRENCY: usize = 4;
pub fn blob_transfer_concurrency() -> crate::Result<NonZeroUsize> {
match env::var(ARTIFACT_TRANSFER_CONCURRENCY_ENV) {
Ok(value) => {
let concurrency = parse_blob_transfer_concurrency(Some(&value))?;
tracing::debug!(
concurrency = concurrency.get(),
variable = ARTIFACT_TRANSFER_CONCURRENCY_ENV,
"Using configured Artifact transfer concurrency"
);
Ok(concurrency)
}
Err(env::VarError::NotPresent) => parse_blob_transfer_concurrency(None),
Err(env::VarError::NotUnicode(_)) => anyhow::bail!(
"{ARTIFACT_TRANSFER_CONCURRENCY_ENV} must be a positive integer encoded as UTF-8"
),
}
}
fn parse_blob_transfer_concurrency(value: Option<&str>) -> crate::Result<NonZeroUsize> {
match value {
Some(value) => value.parse::<NonZeroUsize>().with_context(|| {
format!("{ARTIFACT_TRANSFER_CONCURRENCY_ENV} must be a positive integer, got {value:?}")
}),
None => Ok(NonZeroUsize::new(DEFAULT_ARTIFACT_TRANSFER_CONCURRENCY)
.expect("default Artifact transfer concurrency must be positive")),
}
}
pub async fn bounded_map<I, T, O, F, Fut>(
items: I,
concurrency: NonZeroUsize,
f: F,
) -> crate::Result<Vec<O>>
where
I: IntoIterator<Item = T>,
F: FnMut(T) -> Fut,
Fut: std::future::Future<Output = crate::Result<O>>,
{
stream::iter(items)
.map(f)
.buffer_unordered(concurrency.get())
.try_collect()
.await
}
pub struct RemoteTransport {
runtime: Runtime,
client: Client,
auth: RegistryAuth,
}
#[derive(Debug, thiserror::Error)]
#[error(
"OMMX_BASIC_AUTH_DOMAIN={domain} is set (matches target {domain}), \
but OMMX_BASIC_AUTH_USERNAME={username_state} and \
OMMX_BASIC_AUTH_PASSWORD={password_state}. Both variables are required \
for the env-var auth override; unset OMMX_BASIC_AUTH_DOMAIN to fall back \
to ~/.docker/config.json instead."
)]
pub struct InvalidAuthenticationConfiguration {
domain: String,
username_state: &'static str,
password_state: &'static str,
}
#[derive(Debug, thiserror::Error)]
pub enum InvalidRemoteResponse {
#[error(
"Registry reported Content-Length {content_length} for blob {digest}, \
but the manifest descriptor declares size {expected_size}"
)]
ContentLengthMismatch {
digest: String,
content_length: u64,
expected_size: u64,
},
#[error("Pulled blob size overflowed u64 for {digest}")]
SizeOverflow { digest: String },
#[error(
"Pulled blob bytes for {digest} exceed declared size {expected_size}; \
the registry served {actual_size} bytes"
)]
BlobTooLarge {
digest: String,
expected_size: u64,
actual_size: u64,
},
#[error("Blob digest verification failed for {digest}")]
BlobDigestMismatch {
digest: String,
#[source]
source: io::Error,
},
}
#[derive(Debug, thiserror::Error)]
#[error("Remote blob stream failed for {digest}")]
pub struct RemoteTransportFailure {
digest: String,
#[source]
source: io::Error,
}
fn blob_stream_error(digest: &str, source: io::Error) -> crate::Error {
let is_digest_error = error_contains_digest_error(&source);
if is_digest_error {
crate::error!(InvalidRemoteResponse::BlobDigestMismatch {
digest: digest.to_owned(),
source,
})
} else {
crate::error!(RemoteTransportFailure {
digest: digest.to_owned(),
source,
})
}
}
fn error_contains_digest_error(error: &(dyn std::error::Error + 'static)) -> bool {
if error.downcast_ref::<DigestError>().is_some() {
return true;
}
if let Some(io_error) = error.downcast_ref::<io::Error>() {
if io_error
.get_ref()
.is_some_and(|inner| error_contains_digest_error(inner))
{
return true;
}
}
error.source().is_some_and(error_contains_digest_error)
}
impl RemoteTransport {
pub fn new(image_name: &crate::artifact::ImageRef) -> crate::Result<Self> {
let runtime = RuntimeBuilder::new_current_thread()
.enable_all()
.build()
.context("Failed to build tokio runtime for OCI remote transport")?;
let config = ClientConfig {
protocol: protocol_for(image_name.registry()),
..ClientConfig::default()
};
let client = Client::new(config);
let auth = resolve_auth(image_name.registry())?;
Ok(Self {
runtime,
client,
auth,
})
}
pub fn auth_for(
&self,
image_name: &crate::artifact::ImageRef,
operation: RegistryOperation,
) -> crate::Result<()> {
let reference = to_reference(image_name);
self.runtime
.block_on(self.client.auth(reference, &self.auth, operation))
.with_context(|| format!("Failed to authenticate against {reference}"))?;
Ok(())
}
pub fn auth(&self, image_name: &crate::artifact::ImageRef) -> crate::Result<()> {
self.auth_for(image_name, RegistryOperation::Push)
}
pub async fn blob_exists_async(
&self,
image_name: &crate::artifact::ImageRef,
digest: &str,
) -> crate::Result<bool> {
let reference = to_reference(image_name);
self.client
.blob_exists(reference, digest)
.await
.with_context(|| format!("Failed to check blob {digest} in {reference}"))
}
pub async fn push_blob_async(
&self,
image_name: &crate::artifact::ImageRef,
digest: &str,
bytes: Vec<u8>,
) -> crate::Result<()> {
let reference = to_reference(image_name);
self.client
.push_blob(reference, bytes, digest)
.await
.with_context(|| format!("Failed to push blob {digest} to {reference}"))?;
Ok(())
}
pub fn push_manifest_bytes(
&self,
image_name: &crate::artifact::ImageRef,
manifest_bytes: Vec<u8>,
content_type: &str,
) -> crate::Result<()> {
let reference = to_reference(image_name);
let header: HeaderValue = content_type
.parse()
.with_context(|| format!("Invalid Content-Type {content_type}"))?;
self.runtime
.block_on(
self.client
.push_manifest_raw(reference, manifest_bytes, header),
)
.with_context(|| format!("Failed to push manifest to {reference}"))?;
Ok(())
}
pub fn pull_manifest_raw(
&self,
image_name: &crate::artifact::ImageRef,
accepted_media_types: &[&str],
) -> crate::Result<(Vec<u8>, String)> {
let reference = to_reference(image_name);
let (bytes, digest) = self
.runtime
.block_on(
self.client
.pull_manifest_raw(reference, &self.auth, accepted_media_types),
)
.with_context(|| format!("Failed to pull manifest from {reference}"))?;
Ok((bytes.to_vec(), digest))
}
pub async fn pull_blob_to_vec_async(
&self,
image_name: &crate::artifact::ImageRef,
digest: &str,
expected_size: u64,
) -> crate::Result<Vec<u8>> {
let reference = to_reference(image_name);
let resp = self
.client
.pull_blob_stream(reference, digest)
.await
.with_context(|| format!("Failed to pull blob {digest} from {reference}"))?;
if let Some(content_length) = resp.content_length {
if content_length != expected_size {
return Err(crate::error!(
InvalidRemoteResponse::ContentLengthMismatch {
digest: digest.to_owned(),
content_length,
expected_size,
}
));
}
}
let prealloc = expected_size.min(BLOB_PREALLOC_CAP_BYTES) as usize;
let mut buf = Vec::with_capacity(prealloc);
let mut accumulated: u64 = 0;
let mut stream = resp.stream;
while let Some(chunk) = stream
.try_next()
.await
.map_err(|source| blob_stream_error(digest, source))?
{
accumulated = accumulated.checked_add(chunk.len() as u64).ok_or_else(|| {
crate::error!(InvalidRemoteResponse::SizeOverflow {
digest: digest.to_owned(),
})
})?;
if accumulated > expected_size {
return Err(crate::error!(InvalidRemoteResponse::BlobTooLarge {
digest: digest.to_owned(),
expected_size,
actual_size: accumulated,
}));
}
buf.extend_from_slice(&chunk);
}
Ok(buf)
}
pub fn block_on<F: std::future::Future>(&self, future: F) -> F::Output {
self.runtime.block_on(future)
}
}
const BLOB_PREALLOC_CAP_BYTES: u64 = 256 * 1024 * 1024;
fn to_reference(image_name: &crate::artifact::ImageRef) -> &Reference {
image_name.as_inner()
}
fn protocol_for(registry: &str) -> ClientProtocol {
let host = registry
.split_once(':')
.map(|(host, _)| host)
.unwrap_or(registry);
if host == "localhost" || host.starts_with("127.") || host.starts_with("::1") {
ClientProtocol::Http
} else {
ClientProtocol::Https
}
}
fn resolve_auth(registry_key: &str) -> crate::Result<RegistryAuth> {
if let Some(auth) = auth_from_env(registry_key)? {
return Ok(auth);
}
if let Some(auth) = auth_from_docker_config(registry_key) {
return Ok(auth);
}
tracing::debug!("No credentials resolved for {registry_key}; using anonymous auth");
Ok(RegistryAuth::Anonymous)
}
struct EnvCredentials {
domain: Option<String>,
username: Option<String>,
password: Option<String>,
}
fn auth_from_env(registry_key: &str) -> crate::Result<Option<RegistryAuth>> {
classify_env_credentials(
registry_key,
EnvCredentials {
domain: env::var("OMMX_BASIC_AUTH_DOMAIN").ok(),
username: env::var("OMMX_BASIC_AUTH_USERNAME").ok(),
password: env::var("OMMX_BASIC_AUTH_PASSWORD").ok(),
},
)
}
fn classify_env_credentials(
registry_key: &str,
creds: EnvCredentials,
) -> crate::Result<Option<RegistryAuth>> {
let Some(domain) = creds.domain else {
return Ok(None);
};
if domain != registry_key {
tracing::debug!(
"OMMX_BASIC_AUTH_DOMAIN={domain} does not match target {registry_key}; \
falling through to docker config"
);
return Ok(None);
}
match (creds.username, creds.password) {
(Some(username), Some(password)) => {
tracing::info!(
"Using OMMX_BASIC_AUTH credentials for {registry_key} (user {username})"
);
Ok(Some(RegistryAuth::Basic(username, password)))
}
(u, p) => {
let username_state = if u.is_some() { "set" } else { "unset" };
let password_state = if p.is_some() { "set" } else { "unset" };
Err(crate::error!(InvalidAuthenticationConfiguration {
domain,
username_state,
password_state,
}))
}
}
}
fn auth_from_docker_config(hostname: &str) -> Option<RegistryAuth> {
classify_docker_credential(hostname, docker_credential::get_credential(hostname))
}
fn classify_docker_credential(
hostname: &str,
result: std::result::Result<DockerCredential, CredentialRetrievalError>,
) -> Option<RegistryAuth> {
match result {
Ok(DockerCredential::UsernamePassword(username, password)) => {
tracing::info!("Using docker config Basic auth for {hostname} (user {username})");
Some(RegistryAuth::Basic(username, password))
}
Ok(DockerCredential::IdentityToken(token)) => {
tracing::info!("Using docker config identity token for {hostname}");
Some(RegistryAuth::Bearer(token))
}
Err(
CredentialRetrievalError::ConfigNotFound
| CredentialRetrievalError::NoCredentialConfigured,
) => None,
Err(_) => {
tracing::warn!(
"Failed to read docker credential for {hostname}; falling through to anonymous"
);
None
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::artifact::{remote_error, ImageRef, RemoteArtifactError};
use std::sync::{
atomic::{AtomicUsize, Ordering},
Arc,
};
#[test]
fn bounded_map_limits_in_flight_operations() {
let runtime = RuntimeBuilder::new_current_thread()
.enable_time()
.build()
.unwrap();
let in_flight = Arc::new(AtomicUsize::new(0));
let maximum = Arc::new(AtomicUsize::new(0));
let output = runtime
.block_on(bounded_map(0..12, NonZeroUsize::new(3).unwrap(), |item| {
let in_flight = Arc::clone(&in_flight);
let maximum = Arc::clone(&maximum);
async move {
let current = in_flight.fetch_add(1, Ordering::SeqCst) + 1;
maximum.fetch_max(current, Ordering::SeqCst);
tokio::time::sleep(std::time::Duration::from_millis(2)).await;
in_flight.fetch_sub(1, Ordering::SeqCst);
Ok(item)
}
}))
.unwrap();
assert_eq!(output.len(), 12);
assert_eq!(maximum.load(Ordering::SeqCst), 3);
}
#[test]
fn bounded_map_returns_a_concurrent_operation_error() {
let runtime = RuntimeBuilder::new_current_thread().build().unwrap();
let error = runtime
.block_on(bounded_map(
0..4,
NonZeroUsize::new(2).unwrap(),
|item| async move {
anyhow::ensure!(item != 2, "blob sha256:failed failed");
Ok(item)
},
))
.unwrap_err();
assert!(error.to_string().contains("sha256:failed"));
}
#[test]
fn artifact_transfer_concurrency_uses_conservative_default() {
assert_eq!(
parse_blob_transfer_concurrency(None).unwrap().get(),
DEFAULT_ARTIFACT_TRANSFER_CONCURRENCY
);
}
#[test]
fn artifact_transfer_concurrency_accepts_positive_override() {
assert_eq!(parse_blob_transfer_concurrency(Some("8")).unwrap().get(), 8);
}
#[test]
fn artifact_transfer_concurrency_rejects_invalid_override() {
for value in ["", "0", "-1", "four"] {
let error = parse_blob_transfer_concurrency(Some(value)).unwrap_err();
assert!(error
.to_string()
.contains(ARTIFACT_TRANSFER_CONCURRENCY_ENV));
}
}
fn assert_basic(auth: Option<RegistryAuth>, expected_user: &str, expected_pass: &str) {
match auth {
Some(RegistryAuth::Basic(u, p)) => {
assert_eq!(u, expected_user);
assert_eq!(p, expected_pass);
}
other => panic!("expected Basic({expected_user}, _), got {other:?}"),
}
}
fn assert_bearer(auth: Option<RegistryAuth>, expected_token: &str) {
match auth {
Some(RegistryAuth::Bearer(t)) => assert_eq!(t, expected_token),
other => panic!("expected Bearer(_), got {other:?}"),
}
}
#[test]
fn classify_username_password_to_basic() {
let auth = classify_docker_credential(
"ghcr.io",
Ok(DockerCredential::UsernamePassword(
"alice".to_string(),
"secret".to_string(),
)),
);
assert_basic(auth, "alice", "secret");
}
#[test]
fn classify_identity_token_to_bearer() {
let auth = classify_docker_credential(
"ghcr.io",
Ok(DockerCredential::IdentityToken("tok-123".to_string())),
);
assert_bearer(auth, "tok-123");
}
#[test]
fn classify_missing_credentials_falls_through() {
for err in [
CredentialRetrievalError::ConfigNotFound,
CredentialRetrievalError::NoCredentialConfigured,
] {
assert!(classify_docker_credential("ghcr.io", Err(err)).is_none());
}
}
#[test]
fn classify_helper_failure_falls_through() {
let auth = classify_docker_credential(
"ghcr.io",
Err(CredentialRetrievalError::HelperFailure {
helper: "docker-credential-broken".to_string(),
stdout: "{\"Secret\": \"leaked-token\"}".to_string(),
stderr: "AWS_SECRET_ACCESS_KEY=should-not-appear-in-logs".to_string(),
}),
);
assert!(auth.is_none());
}
#[test]
fn registry_string_includes_port_when_present() {
let with_port = ImageRef::parse("localhost:5000/ommx/native-push:tag1").unwrap();
assert_eq!(with_port.registry(), "localhost:5000");
let no_port = ImageRef::parse("ghcr.io/jij-inc/ommx:tag1").unwrap();
assert_eq!(no_port.registry(), "ghcr.io");
}
#[test]
fn protocol_for_picks_http_for_localhost_variants() {
for registry in ["localhost", "localhost:5000", "127.0.0.1", "127.0.0.1:5000"] {
assert!(
matches!(protocol_for(registry), ClientProtocol::Http),
"expected HTTP for local registry {registry}",
);
}
for registry in ["ghcr.io", "registry.example.com:443"] {
assert!(
matches!(protocol_for(registry), ClientProtocol::Https),
"expected HTTPS for remote registry {registry}",
);
}
}
fn env(domain: Option<&str>, username: Option<&str>, password: Option<&str>) -> EnvCredentials {
EnvCredentials {
domain: domain.map(str::to_string),
username: username.map(str::to_string),
password: password.map(str::to_string),
}
}
#[test]
fn env_creds_no_domain_falls_through() {
let out = classify_env_credentials("ghcr.io", env(None, None, None)).unwrap();
assert!(out.is_none());
let out = classify_env_credentials("ghcr.io", env(None, Some("u"), Some("p"))).unwrap();
assert!(out.is_none());
}
#[test]
fn env_creds_domain_mismatch_falls_through() {
let out = classify_env_credentials(
"ghcr.io",
env(Some("registry.example.com"), Some("u"), Some("p")),
)
.unwrap();
assert!(out.is_none());
}
#[test]
fn env_creds_full_override_produces_basic() {
let out = classify_env_credentials(
"ghcr.io",
env(Some("ghcr.io"), Some("alice"), Some("secret")),
)
.unwrap();
assert_basic(out, "alice", "secret");
}
#[test]
fn env_creds_partial_override_errors() {
let image = ImageRef::parse("ghcr.io/jij-inc/ommx:latest").unwrap();
for (u, p) in [(Some("alice"), None), (None, Some("secret")), (None, None)] {
let err = classify_env_credentials("ghcr.io", env(Some("ghcr.io"), u, p))
.expect_err("partial OMMX_BASIC_AUTH_* should be an error");
let msg = err.to_string();
assert!(msg.contains("OMMX_BASIC_AUTH_DOMAIN=ghcr.io"));
assert!(msg.contains("USERNAME="));
assert!(msg.contains("PASSWORD="));
assert!(matches!(
remote_error::classify_manifest(&image, err),
RemoteArtifactError::Authentication { .. }
));
}
}
#[test]
fn invalid_remote_response_is_classified_without_message_matching() {
let image = ImageRef::parse("ghcr.io/jij-inc/ommx:latest").unwrap();
let source = crate::error!(InvalidRemoteResponse::ContentLengthMismatch {
digest: "sha256:deadbeef".to_string(),
content_length: 2,
expected_size: 1,
});
assert!(matches!(
remote_error::classify_manifest(&image, source),
RemoteArtifactError::InvalidArtifact { .. }
));
}
#[test]
fn blob_stream_io_is_classified_as_transport() {
let image = ImageRef::parse("ghcr.io/jij-inc/ommx:latest").unwrap();
let source = blob_stream_error(
"sha256:deadbeef",
io::Error::new(io::ErrorKind::ConnectionReset, "connection reset"),
);
assert!(matches!(
remote_error::classify_blob(&image, source),
RemoteArtifactError::Transport { .. }
));
}
}