pub(crate) const EXPERIMENTAL: bool = false;
use crate::backend::backend_type::BackendType;
use crate::backend::options::BackendOptions;
use crate::backend::platform_target::PlatformTarget;
use crate::backend::static_helpers::{
ArchiveLayout, get_filename_from_url, install_artifact, template_string,
template_string_for_target, verify_artifact,
};
use crate::backend::version_list;
use crate::backend::{Backend, VersionInfo, runtime_path_for_install_path};
use crate::cli::args::BackendArg;
use crate::config::{Config, Settings};
use crate::file;
use crate::hash;
use crate::install_context::InstallContext;
use crate::lockfile::PlatformInfo;
use crate::toolset::{ToolVersion, ToolVersionOptions};
use crate::ui::progress_report::SingleReport;
use async_trait::async_trait;
use aws_config::BehaviorVersion;
use aws_sdk_s3::Client as S3Client;
use eyre::{Result, bail, eyre};
use regex::Regex;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use tokio::sync::OnceCell;
use url::Url;
#[derive(Debug, Clone)]
struct S3Url {
bucket: String,
key: String,
}
impl S3Url {
fn parse(url_str: &str) -> Result<Self> {
let url = Url::parse(url_str).map_err(|e| eyre!("Invalid S3 URL: {e}"))?;
if url.scheme() != "s3" {
bail!("URL must use s3:// scheme, got: {}", url.scheme());
}
let bucket = url
.host_str()
.ok_or_else(|| eyre!("S3 URL must include bucket name"))?
.to_string();
if bucket.is_empty() {
bail!("S3 URL must include bucket name");
}
let key = url.path().trim_start_matches('/').to_string();
Ok(Self { bucket, key })
}
}
#[derive(Debug)]
pub(crate) struct S3Backend {
ba: Arc<BackendArg>,
client: OnceCell<S3Client>,
}
#[derive(Debug, Clone, Copy)]
struct S3Options<'a> {
values: BackendOptions<'a>,
}
impl<'a> S3Options<'a> {
fn new(raw: &'a ToolVersionOptions) -> Self {
Self {
values: BackendOptions::new(raw),
}
}
fn raw(&self) -> &'a ToolVersionOptions {
self.values.raw()
}
fn url(&self) -> Option<String> {
self.values.platform_string("url")
}
fn checksum(&self) -> Option<String> {
self.values.platform_string("checksum")
}
fn url_for_target(&self, target: &PlatformTarget) -> Option<String> {
self.values.platform_string_for_target("url", target)
}
fn checksum_for_target(&self, target: &PlatformTarget) -> Option<String> {
self.values.platform_string_for_target("checksum", target)
}
fn bin_path(&self) -> Option<String> {
self.values.platform_string("bin_path")
}
fn region(&self) -> Option<String> {
self.values.platform_string("region")
}
fn endpoint(&self) -> Option<String> {
self.values.platform_string("endpoint")
}
fn version_list_url(&self) -> Option<String> {
self.values.platform_string("version_list_url")
}
fn version_prefix(&self) -> Option<String> {
self.values.platform_string("version_prefix")
}
fn version_regex(&self) -> Option<String> {
self.values.platform_string("version_regex")
}
fn version_json_path(&self) -> Option<String> {
self.values.platform_string("version_json_path")
}
fn version_expr(&self) -> Option<String> {
self.values.platform_string("version_expr")
}
fn url_platforms(&self) -> Vec<String> {
self.values.available_platforms_with_key("url")
}
}
impl S3Backend {
pub(crate) fn from_arg(ba: BackendArg) -> Self {
Self {
ba: Arc::new(ba),
client: OnceCell::new(),
}
}
async fn get_client(&self, opts: &S3Options<'_>) -> Result<&S3Client> {
self.client
.get_or_try_init(|| async {
let region = opts.region();
let endpoint = opts.endpoint();
create_s3_client(region.as_deref(), endpoint.as_deref()).await
})
.await
}
fn resolve_url(&self, tv: &ToolVersion, opts: &S3Options<'_>) -> Result<String> {
let url_template = opts.url().ok_or_else(|| self.missing_url_error(opts))?;
Ok(template_string(&url_template, tv))
}
fn missing_url_error(&self, opts: &S3Options<'_>) -> eyre::Report {
let platform_key = self.get_platform_key();
let available = opts.url_platforms();
if available.is_empty() {
eyre::eyre!("S3 backend requires 'url' option")
} else {
eyre::eyre!(
"No URL for platform {platform_key}. Available: {}. \
Provide 'url' or add 'platforms.{platform_key}.url'",
available.join(", ")
)
}
}
fn lock_url_for_target(
&self,
opts: &S3Options<'_>,
tv: &ToolVersion,
target: &PlatformTarget,
) -> Option<String> {
opts.url_for_target(target)
.map(|template| template_string_for_target(&template, tv, target))
}
async fn download_object(
&self,
client: &S3Client,
s3_url: &S3Url,
dest: &Path,
pr: Option<&dyn SingleReport>,
) -> Result<()> {
debug!(
"Downloading s3://{}/{} to {}",
s3_url.bucket,
s3_url.key,
dest.display()
);
let resp = client
.get_object()
.bucket(&s3_url.bucket)
.key(&s3_url.key)
.send()
.await
.map_err(|e| handle_s3_error(e, &s3_url.bucket, &s3_url.key))?;
let content_length = resp.content_length().unwrap_or(0) as u64;
if let Some(pr) = pr {
pr.set_length(content_length);
}
let body = resp
.body
.collect()
.await
.map_err(|e| eyre!("Failed to read S3 response body: {e}"))?;
let bytes = body.into_bytes();
let tmp_path = dest.with_extension("tmp");
file::write(&tmp_path, &bytes)?;
std::fs::rename(&tmp_path, dest)?;
if let Some(pr) = pr {
pr.set_position(content_length);
}
Ok(())
}
async fn fetch_versions_from_manifest(
&self,
client: &S3Client,
manifest_url: &str,
opts: &S3Options<'_>,
) -> Result<Vec<String>> {
let s3_url = S3Url::parse(manifest_url)?;
let tmp_dir = tempfile::tempdir()?;
let tmp_path = tmp_dir.path().join("versions_manifest");
self.download_object(client, &s3_url, &tmp_path, None)
.await?;
let content = file::read_to_string(&tmp_path)?;
let regex = opts.version_regex();
let json_path = opts.version_json_path();
let version_expr = opts.version_expr();
version_list::parse_version_list(
&content,
regex.as_deref(),
json_path.as_deref(),
version_expr.as_deref(),
)
}
async fn fetch_versions_from_listing(
&self,
client: &S3Client,
bucket: &str,
prefix: &str,
version_regex: &str,
) -> Result<Vec<String>> {
let regex =
Regex::new(version_regex).map_err(|e| eyre!("Invalid version_regex pattern: {e}"))?;
let mut versions = Vec::new();
let mut continuation_token: Option<String> = None;
loop {
let mut request = client.list_objects_v2().bucket(bucket).prefix(prefix);
if let Some(token) = continuation_token {
request = request.continuation_token(token);
}
let response = request
.send()
.await
.map_err(|e| handle_s3_error(e, bucket, prefix))?;
if let Some(contents) = response.contents {
for object in contents {
if let Some(key) = object.key {
if let Some(captures) = regex.captures(&key) {
let version = captures
.get(1)
.or_else(|| captures.get(0))
.map(|m| m.as_str().to_string());
if let Some(v) = version
&& !versions.contains(&v)
{
versions.push(v);
}
}
}
}
}
if response.is_truncated == Some(true) {
continuation_token = response.next_continuation_token;
} else {
break;
}
}
Ok(versions)
}
async fn fetch_versions(&self, config: &Arc<Config>) -> Result<Vec<String>> {
let raw_opts = config.get_tool_opts_with_overrides(&self.ba).await?;
let opts = S3Options::new(&raw_opts);
if let Some(manifest_url) = opts.version_list_url() {
let client = self.get_client(&opts).await?;
return self
.fetch_versions_from_manifest(client, &manifest_url, &opts)
.await;
}
if let Some(version_prefix) = opts.version_prefix() {
let version_regex = opts
.version_regex()
.unwrap_or_else(|| r"([0-9]+\.[0-9]+\.[0-9]+)".to_string());
let url_template = opts
.url()
.ok_or_else(|| eyre!("S3 backend requires 'url' option for version listing"))?;
let s3_url = S3Url::parse(&url_template)?;
let client = self.get_client(&opts).await?;
return self
.fetch_versions_from_listing(
client,
&s3_url.bucket,
&version_prefix,
&version_regex,
)
.await;
}
Ok(vec![])
}
fn verify_checksum(
&self,
ctx: &InstallContext,
tv: &mut ToolVersion,
file_path: &Path,
) -> Result<()> {
let settings = Settings::get();
let filename = file_path.file_name().unwrap().to_string_lossy();
let lockfile_enabled = settings.lockfile_enabled();
let platform_key = self.get_platform_key();
let platform_info = tv.lock_platforms.entry(platform_key).or_default();
if let Some(checksum) = &platform_info.checksum {
ctx.pr.set_message(format!("checksum {filename}"));
let (algo, check) = checksum
.split_once(':')
.ok_or_else(|| eyre!("Invalid checksum format: {checksum}"))?;
hash::ensure_checksum(file_path, check, Some(ctx.pr.as_ref()), algo)?;
} else if lockfile_enabled {
ctx.pr.set_message(format!("generate checksum {filename}"));
let h = hash::file_hash_blake3(file_path, Some(ctx.pr.as_ref()))?;
platform_info.checksum = Some(format!("blake3:{h}"));
}
if let Some(expected_size) = platform_info.size {
ctx.pr.set_message(format!("verify size {filename}"));
let actual_size = file_path.metadata()?.len();
if actual_size != expected_size {
return Err(eyre!(
"Size mismatch for {filename}: expected {expected_size}, got {actual_size}"
));
}
} else if lockfile_enabled {
platform_info.size = Some(file_path.metadata()?.len());
}
Ok(())
}
}
pub(crate) fn install_time_option_keys() -> Vec<String> {
vec![
"url".into(),
"checksum".into(),
"bin_path".into(),
"version_list_url".into(),
"version_regex".into(),
"version_json_path".into(),
"version_expr".into(),
"version_prefix".into(),
"format".into(),
"region".into(),
"endpoint".into(),
]
}
async fn create_s3_client(region: Option<&str>, endpoint: Option<&str>) -> Result<S3Client> {
let mut config_loader = aws_config::defaults(BehaviorVersion::latest());
if let Some(region) = region {
config_loader = config_loader.region(aws_config::Region::new(region.to_string()));
}
let sdk_config = config_loader.load().await;
let mut s3_config = aws_sdk_s3::config::Builder::from(&sdk_config);
if let Some(endpoint) = endpoint {
s3_config = s3_config.endpoint_url(endpoint).force_path_style(true);
}
Ok(S3Client::from_conf(s3_config.build()))
}
fn handle_s3_error<E: std::fmt::Debug>(err: E, bucket: &str, key: &str) -> eyre::Report {
let err_str = format!("{err:?}");
if err_str.contains("NoSuchKey") {
eyre!("S3 object not found: s3://{bucket}/{key}. Check the URL and version.")
} else if err_str.contains("NoSuchBucket") {
eyre!("S3 bucket not found: {bucket}. Check the bucket name.")
} else if err_str.contains("AccessDenied") || err_str.contains("Forbidden") {
eyre!(
"Access denied to S3 bucket '{bucket}'. Check your AWS credentials and IAM permissions.\n\
Ensure AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are set, or use IAM roles."
)
} else if err_str.contains("InvalidAccessKeyId") {
eyre!("Invalid AWS access key. Check your AWS_ACCESS_KEY_ID environment variable.")
} else if err_str.contains("SignatureDoesNotMatch") {
eyre!("AWS signature mismatch. Check your AWS_SECRET_ACCESS_KEY environment variable.")
} else if err_str.contains("timeout") || err_str.contains("Timeout") {
eyre!("S3 request timed out. Check your network connection and endpoint URL.")
} else {
eyre!("S3 error: {err:?}")
}
}
#[async_trait]
impl Backend for S3Backend {
fn get_type(&self) -> BackendType {
BackendType::S3
}
fn ba(&self) -> &Arc<BackendArg> {
&self.ba
}
fn mark_prereleases_from_version_pattern(&self) -> bool {
true
}
fn remote_version_listing_tool_option_keys(&self) -> &'static [&'static str] {
&[
"version_list_url",
"version_regex",
"version_json_path",
"version_expr",
"version_prefix",
"url",
"region",
"endpoint",
]
}
async fn install_operation_count(&self, tv: &ToolVersion, _ctx: &InstallContext) -> usize {
let raw_opts = tv.request.options();
let opts = S3Options::new(&raw_opts);
super::http_install_operation_count(opts.checksum().is_some(), &self.get_platform_key(), tv)
}
async fn resolve_lock_info(
&self,
tv: &ToolVersion,
target: &PlatformTarget,
) -> Result<PlatformInfo> {
let raw_opts = tv.request.options();
let opts = S3Options::new(&raw_opts);
let Some(url) = self.lock_url_for_target(&opts, tv, target) else {
return Err(eyre!(
"no URL configured for {} on {}; skipping",
self.ba.full(),
target.to_key()
));
};
Ok(PlatformInfo {
url: Some(url),
checksum: opts.checksum_for_target(target),
..Default::default()
})
}
async fn _list_remote_versions(&self, config: &Arc<Config>) -> Result<Vec<VersionInfo>> {
let versions = self.fetch_versions(config).await?;
Ok(versions
.into_iter()
.map(|v| VersionInfo {
version: v,
..Default::default()
})
.collect())
}
async fn verify_install_feasible(&self, _ctx: &InstallContext, tv: &ToolVersion) -> Result<()> {
let raw_opts = tv.request.options();
let opts = S3Options::new(&raw_opts);
match opts.url() {
Some(_) => Ok(()),
None => Err(self.missing_url_error(&opts)),
}
}
async fn install_version_(
&self,
ctx: &InstallContext,
mut tv: ToolVersion,
) -> Result<ToolVersion> {
let raw_opts = tv.request.options();
let opts = S3Options::new(&raw_opts);
let url = self.resolve_url(&tv, &opts)?;
let s3_url = S3Url::parse(&url)?;
let client = self.get_client(&opts).await?;
let filename = get_filename_from_url(&url);
let file_path = tv.download_path().join(&filename);
let platform_key = self.get_platform_key();
tv.lock_platforms
.entry(platform_key.clone())
.or_default()
.url = Some(url.clone());
let settings = Settings::get();
let lockfile_enabled = settings.lockfile_enabled();
let has_lockfile_checksum = tv
.lock_platforms
.get(&platform_key)
.and_then(|p| p.checksum.as_ref())
.is_some();
ctx.pr.set_message(format!("download {filename}"));
file::create_dir_all(tv.download_path())?;
self.download_object(client, &s3_url, &file_path, Some(ctx.pr.as_ref()))
.await?;
if opts.checksum().is_some() {
ctx.pr.next_operation();
}
verify_artifact(&tv, &file_path, opts.raw(), Some(ctx.pr.as_ref()))?;
if lockfile_enabled || has_lockfile_checksum {
ctx.pr.next_operation();
}
self.verify_checksum(ctx, &mut tv, &file_path)?;
ctx.pr.next_operation();
ctx.pr.set_message("extract".into());
install_artifact(
&tv,
&file_path,
opts.raw(),
ArchiveLayout::Guessed,
Some(ctx.pr.as_ref()),
)?;
Ok(tv)
}
async fn list_bin_paths(
&self,
_config: &Arc<Config>,
tv: &ToolVersion,
) -> Result<Vec<PathBuf>> {
let raw_opts = tv.request.options();
let opts = S3Options::new(&raw_opts);
if let Some(bin_path_template) = opts.bin_path() {
let bin_path = template_string(&bin_path_template, tv);
return Ok(vec![runtime_path_for_install_path(
tv,
tv.install_path().join(bin_path),
)]);
}
let bin_dir = tv.install_path().join("bin");
if bin_dir.exists() {
return Ok(vec![runtime_path_for_install_path(tv, bin_dir)]);
}
let mut paths = Vec::new();
if let Ok(entries) = std::fs::read_dir(tv.install_path()) {
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
let sub_bin = path.join("bin");
if sub_bin.exists() {
paths.push(sub_bin);
}
}
}
}
if paths.is_empty() {
Ok(vec![tv.runtime_path()])
} else {
Ok(paths
.into_iter()
.map(|path| runtime_path_for_install_path(tv, path))
.collect())
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::cli::args::BackendResolution;
use crate::platform::Platform;
use crate::toolset::{ToolRequest, ToolSource};
use indexmap::IndexMap;
fn s3_test_backend() -> S3Backend {
S3Backend::from_arg(BackendArg::new_raw(
"s3-xtool".to_string(),
Some("s3:xtool".to_string()),
"xtool".to_string(),
None,
BackendResolution::new(true),
))
}
fn s3_test_tv(version: &str, options: ToolVersionOptions) -> ToolVersion {
let backend = Arc::new(BackendArg::new_raw(
"s3-xtool".to_string(),
Some("s3:xtool".to_string()),
"xtool".to_string(),
None,
BackendResolution::new(true),
));
let request =
ToolRequest::new_with_options(backend, version, options, ToolSource::Argument).unwrap();
ToolVersion::new(request, version.to_string())
}
fn target(key: &str) -> PlatformTarget {
PlatformTarget::new(Platform::parse(key).unwrap())
}
#[test]
fn test_lock_url_for_target_platforms_table() {
let mut opts = IndexMap::new();
opts.insert(
"platforms".to_string(),
toml::from_str::<toml::Value>(
r#"
[linux-x64]
url = "s3://bucket/tools/xtool-{{version}}-linux-x64.tar.gz"
[macos-arm64]
url = "s3://bucket/tools/xtool-{{version}}-macos-arm64.tar.gz"
checksum = "sha256:abc123"
"#,
)
.unwrap(),
);
let tool_opts = ToolVersionOptions {
opts: opts.into(),
..Default::default()
};
let backend = s3_test_backend();
let tv = s3_test_tv("1.0.0", tool_opts);
let raw_opts = tv.request.options();
let opts = S3Options::new(&raw_opts);
assert_eq!(
backend.lock_url_for_target(&opts, &tv, &target("linux-x64")),
Some("s3://bucket/tools/xtool-1.0.0-linux-x64.tar.gz".to_string())
);
assert_eq!(
backend.lock_url_for_target(&opts, &tv, &target("macos-arm64")),
Some("s3://bucket/tools/xtool-1.0.0-macos-arm64.tar.gz".to_string())
);
assert_eq!(
backend.lock_url_for_target(&opts, &tv, &target("windows-x64")),
None
);
assert_eq!(
opts.checksum_for_target(&target("macos-arm64")),
Some("sha256:abc123".to_string())
);
assert_eq!(opts.checksum_for_target(&target("linux-x64")), None);
}
#[test]
fn test_lock_url_for_target_base_template() {
let mut opts = IndexMap::new();
opts.insert(
"url".to_string(),
toml::Value::String(
"s3://bucket/tools/xtool-{{ version }}-{{ os() }}-{{ arch() }}.tar.gz".to_string(),
),
);
let tool_opts = ToolVersionOptions {
opts: opts.into(),
..Default::default()
};
let backend = s3_test_backend();
let tv = s3_test_tv("2.1.0", tool_opts);
let raw_opts = tv.request.options();
let opts = S3Options::new(&raw_opts);
assert_eq!(
backend.lock_url_for_target(&opts, &tv, &target("linux-arm64")),
Some("s3://bucket/tools/xtool-2.1.0-linux-arm64.tar.gz".to_string())
);
assert_eq!(
backend.lock_url_for_target(&opts, &tv, &target("macos-x64")),
Some("s3://bucket/tools/xtool-2.1.0-macos-x64.tar.gz".to_string())
);
}
#[test]
fn test_s3_url_parse_basic() {
let url = S3Url::parse("s3://my-bucket/path/to/file.tar.gz").unwrap();
assert_eq!(url.bucket, "my-bucket");
assert_eq!(url.key, "path/to/file.tar.gz");
}
#[test]
fn test_s3_url_parse_with_query_params() {
let url = S3Url::parse("s3://my-bucket/path/to/file.tar.gz?region=us-west-2").unwrap();
assert_eq!(url.bucket, "my-bucket");
assert_eq!(url.key, "path/to/file.tar.gz");
}
#[test]
fn test_s3_url_parse_root_key() {
let url = S3Url::parse("s3://bucket/file.tar.gz").unwrap();
assert_eq!(url.bucket, "bucket");
assert_eq!(url.key, "file.tar.gz");
}
#[test]
fn test_s3_url_parse_deep_path() {
let url = S3Url::parse("s3://bucket/path/to/mytool-1.0.0.tar.gz").unwrap();
assert_eq!(url.bucket, "bucket");
assert_eq!(url.key, "path/to/mytool-1.0.0.tar.gz");
}
#[test]
fn test_s3_url_invalid_scheme() {
let result = S3Url::parse("https://bucket/path");
assert!(result.is_err());
}
#[test]
fn test_s3_url_missing_bucket() {
let result = S3Url::parse("s3:///path/to/file");
assert!(result.is_err());
}
#[test]
fn missing_url_error_names_this_platform_and_the_ones_declared() {
let backend = s3_test_backend();
let declared = crate::toolset::parse_tool_options("platforms_linux_x64_url=s3://bucket/t");
let declared = S3Options::new(&declared);
let err = backend.missing_url_error(&declared).to_string();
assert!(err.contains(&backend.get_platform_key()), "{err}");
assert!(err.contains("linux-x64"), "{err}");
let none = ToolVersionOptions::default();
let none = S3Options::new(&none);
assert_eq!(
backend.missing_url_error(&none).to_string(),
"S3 backend requires 'url' option"
);
}
}