use std::io::Read as _;
use std::path::{Path, PathBuf};
use sui_compat::flake::LockedInput;
use sui_compat::flake_ref::FlakeRef;
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum FetchError {
#[error("unsupported input type: {0}")]
UnsupportedType(String),
#[error("missing required field: {0}")]
MissingField(&'static str),
#[error("download failed: {0}")]
Download(String),
#[error("throttled by {url} (HTTP {status}){}", match retry_after {
Some(s) => format!(", retry after {s}s"),
None => String::new(),
})]
Throttled {
url: String,
status: u16,
retry_after: Option<u64>,
},
#[error("not authorized for {url} (HTTP {status}) — check the access token")]
Unauthorized { url: String, status: u16 },
#[error("{url} not found (HTTP 404) — or present but invisible to this credential")]
NotFound { url: String },
#[error("{url} returned HTTP {status}")]
UnexpectedStatus { url: String, status: u16 },
#[error("I/O error: {0}")]
Io(#[from] std::io::Error),
#[error("archive extraction failed: {0}")]
Extract(String),
}
impl FetchError {
#[must_use]
pub fn status(&self) -> Option<u16> {
match self {
Self::Throttled { status, .. }
| Self::Unauthorized { status, .. }
| Self::UnexpectedStatus { status, .. } => Some(*status),
Self::NotFound { .. } => Some(404),
_ => None,
}
}
#[must_use]
pub fn is_throttled(&self) -> bool {
matches!(self, Self::Throttled { .. })
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum FailureKind {
Throttled,
Unauthorized,
NotFound,
UnexpectedStatus,
Transport,
Local,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub struct InputFailure {
pub input: String,
pub kind: FailureKind,
#[serde(skip_serializing_if = "Option::is_none")]
pub status: Option<u16>,
#[serde(skip_serializing_if = "Option::is_none")]
pub retry_after: Option<u64>,
pub recoverable_elsewhere: bool,
pub message: String,
}
impl InputFailure {
#[must_use]
pub fn from_error(input: &str, err: &FetchError) -> Self {
let kind = match err {
FetchError::Throttled { .. } => FailureKind::Throttled,
FetchError::Unauthorized { .. } => FailureKind::Unauthorized,
FetchError::NotFound { .. } => FailureKind::NotFound,
FetchError::UnexpectedStatus { .. } => FailureKind::UnexpectedStatus,
FetchError::Download(_) => FailureKind::Transport,
_ => FailureKind::Local,
};
Self {
input: input.to_string(),
kind,
status: err.status(),
retry_after: match err {
FetchError::Throttled { retry_after, .. } => *retry_after,
_ => None,
},
recoverable_elsewhere: err.is_throttled(),
message: err.to_string(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub struct ArchiveReport {
pub scanned: usize,
pub already_present: usize,
pub fetched: usize,
pub failures: Vec<InputFailure>,
}
impl ArchiveReport {
#[must_use]
pub fn is_complete(&self) -> bool {
self.scanned > 0 && self.failures.is_empty()
}
#[must_use]
pub fn recoverable(&self) -> impl Iterator<Item = &InputFailure> {
self.failures.iter().filter(|f| f.recoverable_elsewhere)
}
}
pub struct InputFetcher {
cache_dir: PathBuf,
}
impl Default for InputFetcher {
fn default() -> Self {
Self::new()
}
}
impl InputFetcher {
#[must_use]
pub fn new() -> Self {
let cache_dir = dirs_cache_dir().join("sui/inputs");
Self { cache_dir }
}
#[must_use]
pub fn with_cache_dir(cache_dir: PathBuf) -> Self {
Self { cache_dir }
}
#[must_use]
pub fn cache_dir(&self) -> &Path {
&self.cache_dir
}
#[must_use]
pub fn is_cached(&self, locked: &LockedInput) -> bool {
self.cache_probe(locked).is_some()
}
fn cache_probe(&self, locked: &LockedInput) -> Option<PathBuf> {
let nar_hash = locked.nar_hash.as_ref()?;
let cached = self.cache_dir.join(sanitize_hash(nar_hash));
if !cached.exists() {
return None;
}
let resolved = find_single_subdir_or_self(&cached);
is_non_empty_dir(&resolved).then_some(resolved)
}
pub fn fetch(&self, locked: &LockedInput) -> Result<PathBuf, FetchError> {
if let Some(resolved) = self.cache_probe(locked) {
return Ok(resolved);
}
if let Some(ref nar_hash) = locked.nar_hash {
let cached = self.cache_dir.join(sanitize_hash(nar_hash));
if cached.exists() {
let _ = std::fs::remove_dir_all(&cached);
}
}
match locked.source_type.as_str() {
"github" => self.fetch_github(locked),
"gitlab" => self.fetch_gitlab(locked),
"sourcehut" => self.fetch_sourcehut(locked),
"path" => Self::fetch_path(locked),
"git" => self.fetch_git(locked),
"tarball" | "file" => self.fetch_tarball(locked),
other => Err(FetchError::UnsupportedType(other.to_string())),
}
}
#[must_use]
pub fn github_archive_url(owner: &str, repo: &str, rev: &str) -> String {
format!("https://github.com/{owner}/{repo}/archive/{rev}.tar.gz")
}
#[must_use]
pub fn gitlab_archive_url(host: Option<&str>, owner: &str, repo: &str, rev: &str) -> String {
let host = host.unwrap_or("gitlab.com");
format!(
"https://{host}/{owner}/{repo}/-/archive/{rev}/{repo}-{rev}.tar.gz"
)
}
#[must_use]
pub fn sourcehut_archive_url(owner: &str, repo: &str, rev: &str) -> String {
let owner_prefix = if owner.starts_with('~') {
owner.to_string()
} else {
format!("~{owner}")
};
format!("https://git.sr.ht/{owner_prefix}/{repo}/archive/{rev}.tar.gz")
}
fn fetch_github(&self, locked: &LockedInput) -> Result<PathBuf, FetchError> {
let owner = locked.owner.as_deref().ok_or(FetchError::MissingField("owner"))?;
let repo = locked.repo.as_deref().ok_or(FetchError::MissingField("repo"))?;
let rev = locked.rev.as_deref().ok_or(FetchError::MissingField("rev"))?;
let url = Self::github_archive_url(owner, repo, rev);
self.fetch_archive(locked, &url, &format!("github-{owner}-{repo}-{rev}"), rev)
}
fn fetch_archive(
&self,
locked: &LockedInput,
url: &str,
cache_key: &str,
rev: &str,
) -> Result<PathBuf, FetchError> {
let dest = self.dest_dir(locked, cache_key);
if is_immutable_rev(rev) && is_non_empty_dir(&dest) {
return Ok(find_single_subdir_or_self(&dest));
}
let staging = staging_path(&dest);
let _ = std::fs::remove_dir_all(&staging);
std::fs::create_dir_all(&staging)?;
let bytes = match download_bytes(url) {
Ok(b) => b,
Err(e) => {
let _ = std::fs::remove_dir_all(&staging);
return Err(e);
}
};
if let Err(e) = extract_tar_gz(&bytes, &staging) {
let _ = std::fs::remove_dir_all(&staging);
return Err(e);
}
publish(&staging, &dest, is_immutable_rev(rev))?;
Ok(find_single_subdir_or_self(&dest))
}
fn fetch_gitlab(&self, locked: &LockedInput) -> Result<PathBuf, FetchError> {
let owner = locked.owner.as_deref().ok_or(FetchError::MissingField("owner"))?;
let repo = locked.repo.as_deref().ok_or(FetchError::MissingField("repo"))?;
let rev = locked.rev.as_deref().ok_or(FetchError::MissingField("rev"))?;
let host = locked.host.as_deref();
let url = Self::gitlab_archive_url(host, owner, repo, rev);
let host_tag = host.unwrap_or("gitlab.com").replace('.', "_");
self.fetch_archive(locked, &url, &format!("gitlab-{host_tag}-{owner}-{repo}-{rev}"), rev)
}
fn fetch_sourcehut(&self, locked: &LockedInput) -> Result<PathBuf, FetchError> {
let owner = locked.owner.as_deref().ok_or(FetchError::MissingField("owner"))?;
let repo = locked.repo.as_deref().ok_or(FetchError::MissingField("repo"))?;
let rev = locked.rev.as_deref().ok_or(FetchError::MissingField("rev"))?;
let url = Self::sourcehut_archive_url(owner, repo, rev);
let sanitized_owner = owner.trim_start_matches('~');
self.fetch_archive(
locked,
&url,
&format!("sourcehut-{sanitized_owner}-{repo}-{rev}"),
rev,
)
}
fn fetch_path(locked: &LockedInput) -> Result<PathBuf, FetchError> {
let path = locked
.path
.as_deref()
.ok_or(FetchError::MissingField("path"))?;
Ok(PathBuf::from(path))
}
fn fetch_git(&self, locked: &LockedInput) -> Result<PathBuf, FetchError> {
let url = locked.url.as_deref().ok_or(FetchError::MissingField("url"))?;
let rev = locked.rev.as_deref().ok_or(FetchError::MissingField("rev"))?;
let short_rev: String = rev.chars().take(12).collect();
let dest = self.dest_dir(locked, &format!("git-{short_rev}"));
let immutable = is_immutable_rev(rev);
if immutable && is_non_empty_dir(&dest) {
return Ok(dest);
}
let staging = staging_path(&dest);
let _ = std::fs::remove_dir_all(&staging);
if let Some(tarball_url) = github_tarball_from_git_url(url, rev) {
std::fs::create_dir_all(&staging)?;
match download_bytes(&tarball_url) {
Ok(bytes) => {
if let Err(e) = extract_tar_gz(&bytes, &staging) {
let _ = std::fs::remove_dir_all(&staging);
return Err(e);
}
publish(&staging, &dest, immutable)?;
return Ok(find_single_subdir_or_self(&dest));
}
Err(e) => {
let _ = std::fs::remove_dir_all(&staging);
tracing::debug!(url = %tarball_url, error = %e, "Tarball fallback failed, trying git CLI");
}
}
}
let status = std::process::Command::new("git")
.args(["clone", "--depth", "1", url])
.arg(&staging)
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status()
.map_err(|e| FetchError::Download(format!(
"git clone failed (git not in PATH?): {e}"
)))?;
if !status.success() {
let _ = std::fs::remove_dir_all(&staging);
return Err(FetchError::Download(format!(
"git clone failed for {url} (exit code: {})",
status.code().unwrap_or(-1)
)));
}
if let Err(e) = crate::git::checkout_rev(&staging, rev) {
let _ = std::fs::remove_dir_all(&staging);
return Err(FetchError::Download(format!("git checkout {rev}: {e}")));
}
publish(&staging, &dest, immutable)?;
Ok(dest)
}
fn fetch_tarball(&self, locked: &LockedInput) -> Result<PathBuf, FetchError> {
let url = locked.url.as_deref().ok_or(FetchError::MissingField("url"))?;
let hash_suffix = locked
.nar_hash
.as_deref()
.map_or_else(|| url_to_safe_name(url), sanitize_hash);
let dest = self.dest_dir(locked, &format!("tarball-{hash_suffix}"));
let immutable = locked.nar_hash.is_some();
if immutable && is_non_empty_dir(&dest) {
return Ok(find_single_subdir_or_self(&dest));
}
let staging = staging_path(&dest);
let _ = std::fs::remove_dir_all(&staging);
std::fs::create_dir_all(&staging)?;
let bytes = match download_bytes(url) {
Ok(b) => b,
Err(e) => {
let _ = std::fs::remove_dir_all(&staging);
return Err(e);
}
};
if let Err(e) = extract_tar_gz(&bytes, &staging) {
let _ = std::fs::remove_dir_all(&staging);
return Err(e);
}
publish(&staging, &dest, immutable)?;
Ok(find_single_subdir_or_self(&dest))
}
fn dest_dir(&self, locked: &LockedInput, fallback: &str) -> PathBuf {
if let Some(ref nar_hash) = locked.nar_hash {
self.cache_dir.join(sanitize_hash(nar_hash))
} else {
self.cache_dir.join(fallback)
}
}
}
fn github_tarball_from_git_url(url: &str, rev: &str) -> Option<String> {
let stripped = url
.strip_prefix("https://github.com/")
.or_else(|| url.strip_prefix("git+https://github.com/"))
.or_else(|| url.strip_prefix("http://github.com/"))?;
let stripped = stripped.strip_suffix(".git").unwrap_or(stripped);
let parts: Vec<&str> = stripped.split('/').collect();
if parts.len() == 2 && !parts[0].is_empty() && !parts[1].is_empty() {
Some(format!(
"https://github.com/{}/{}/archive/{rev}.tar.gz",
parts[0], parts[1]
))
} else {
None
}
}
fn sanitize_hash(hash: &str) -> String {
let mapped = hash.replace(':', "-").replace('/', "_").replace('=', "");
let shaped = !mapped.is_empty()
&& mapped != "."
&& mapped != ".."
&& mapped
.bytes()
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'+' | b'-'));
if shaped {
mapped
} else {
use sha2::Digest as _;
let d = sha2::Sha256::digest(hash.as_bytes());
let mut out = String::with_capacity(2 + 64);
out.push_str("h-");
for b in d {
use std::fmt::Write as _;
let _ = write!(out, "{b:02x}");
}
out
}
}
fn is_immutable_rev(rev: &str) -> bool {
matches!(rev.len(), 40 | 64) && rev.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
}
fn staging_path(dest: &Path) -> PathBuf {
let name = dest
.file_name()
.map_or_else(|| "fetch".to_string(), |n| n.to_string_lossy().into_owned());
let tid = format!("{:?}", std::thread::current().id());
let tid: String = tid.chars().filter(char::is_ascii_digit).collect();
let tmp = [
".",
&name,
".tmp-",
&std::process::id().to_string(),
"-",
&tid,
]
.concat();
dest.parent()
.map_or_else(|| PathBuf::from(&tmp), |p| p.join(&tmp))
}
fn publish(staging: &Path, dest: &Path, immutable: bool) -> Result<(), FetchError> {
if immutable && is_non_empty_dir(dest) {
let _ = std::fs::remove_dir_all(staging);
return Ok(());
}
let aside = with_suffix(staging, ".old");
let _ = std::fs::remove_dir_all(&aside);
let moved_aside = dest.exists() && std::fs::rename(dest, &aside).is_ok();
match std::fs::rename(staging, dest) {
Ok(()) => {
if moved_aside {
let _ = std::fs::remove_dir_all(&aside);
}
Ok(())
}
Err(_) => {
if moved_aside && !dest.exists() {
let _ = std::fs::rename(&aside, dest);
}
let _ = std::fs::remove_dir_all(staging);
let _ = std::fs::remove_dir_all(&aside);
if is_non_empty_dir(dest) {
Ok(())
} else {
Err(FetchError::Extract(
"could not publish the fetched tree and no other process left one".into(),
))
}
}
}
}
fn with_suffix(path: &Path, suffix: &str) -> PathBuf {
let name = path
.file_name()
.map_or_else(|| "x".to_string(), |n| n.to_string_lossy().into_owned());
path.parent().map_or_else(
|| PathBuf::from([&name, suffix].concat()),
|p| p.join([&name, suffix].concat()),
)
}
fn is_non_empty_dir(dir: &Path) -> bool {
std::fs::read_dir(dir)
.ok()
.is_some_and(|mut rd| rd.next().is_some())
}
fn find_single_subdir_or_self(dir: &Path) -> PathBuf {
let entries: Vec<_> = std::fs::read_dir(dir)
.ok()
.into_iter()
.flatten()
.filter_map(|e| e.ok())
.collect();
if entries.len() == 1 && entries[0].path().is_dir() {
entries[0].path()
} else {
dir.to_path_buf()
}
}
fn download_bytes(url: &str) -> Result<Vec<u8>, FetchError> {
let agent: ureq::Agent = ureq::Agent::config_builder()
.http_status_as_error(false)
.build()
.into();
let mut req = agent.get(url);
if let Some(token) = github_token_for_url(url) {
req = req.header("Authorization", &format!("token {token}"));
}
let mut response = req
.call()
.map_err(|e| FetchError::Download(format!("{url}: {e}")))?;
if !response.status().is_success() {
return Err(classify_status(url, &response));
}
response
.body_mut()
.with_config()
.limit(512 * 1024 * 1024)
.read_to_vec()
.map_err(|e| FetchError::Download(format!("{url}: {e}")))
}
fn classify_status<B>(url: &str, response: &ureq::http::Response<B>) -> FetchError {
let status = response.status().as_u16();
let retry_after = retry_after_seconds(response.headers());
let throttled = status == 429 || (status == 403 && retry_after.is_some());
if throttled {
FetchError::Throttled {
url: url.to_string(),
status,
retry_after,
}
} else if status == 404 {
FetchError::NotFound {
url: url.to_string(),
}
} else if status == 401 || status == 403 {
FetchError::Unauthorized {
url: url.to_string(),
status,
}
} else {
FetchError::UnexpectedStatus {
url: url.to_string(),
status,
}
}
}
fn retry_after_seconds(headers: &ureq::http::HeaderMap) -> Option<u64> {
headers
.get("retry-after")?
.to_str()
.ok()?
.trim()
.parse::<u64>()
.ok()
}
fn github_token_for_url(url: &str) -> Option<String> {
if !url.starts_with("https://github.com/")
&& !url.starts_with("https://api.github.com/")
{
return None;
}
if let Ok(t) = std::env::var("GITHUB_TOKEN") {
if !t.is_empty() {
return Some(t);
}
}
if let Ok(cfg) = std::env::var("NIX_CONFIG") {
if let Some(t) = parse_access_tokens(&cfg, "github.com") {
return Some(t);
}
}
if let Some(home) = std::env::var_os("HOME").map(PathBuf::from) {
let nix_conf = home.join(".config/nix/nix.conf");
if let Ok(cfg) = std::fs::read_to_string(&nix_conf) {
if let Some(t) = parse_access_tokens(&cfg, "github.com") {
return Some(t);
}
}
let gh_hosts = home.join(".config/gh/hosts.yml");
if let Ok(yml) = std::fs::read_to_string(&gh_hosts) {
if let Some(t) = parse_gh_hosts_token(&yml, "github.com") {
return Some(t);
}
}
}
None
}
fn parse_access_tokens(cfg: &str, host: &str) -> Option<String> {
for line in cfg.lines() {
let trimmed = line.trim();
if let Some(rest) = trimmed.strip_prefix("access-tokens") {
let rest = rest.trim_start().trim_start_matches('=').trim();
for pair in rest.split_whitespace() {
if let Some((h, t)) = pair.split_once('=') {
if h == host {
return Some(t.to_string());
}
}
}
}
}
None
}
fn parse_gh_hosts_token(yml: &str, host: &str) -> Option<String> {
let mut in_host = false;
for line in yml.lines() {
let raw = line;
let trimmed = raw.trim();
if trimmed.starts_with(host) && trimmed.ends_with(':') {
in_host = true;
continue;
}
if !raw.starts_with(' ') && !raw.starts_with('\t') && !trimmed.is_empty() {
in_host = false;
}
if in_host {
if let Some(rest) = trimmed.strip_prefix("oauth_token:") {
return Some(rest.trim().to_string());
}
}
}
None
}
fn extract_tar_gz(bytes: &[u8], dest: &Path) -> Result<(), FetchError> {
let gz = flate2::read::GzDecoder::new(bytes);
let mut buffered = std::io::BufReader::new(gz);
let mut peek = [0u8; 1];
match buffered.read(&mut peek) {
Ok(0) => {
return Err(FetchError::Extract("empty archive".into()));
}
Err(e) => {
return Err(FetchError::Extract(format!("gzip decompression: {e}")));
}
Ok(_) => {
let cursor = std::io::Cursor::new(peek);
let chain = cursor.chain(buffered);
let mut archive = tar::Archive::new(chain);
archive
.unpack(dest)
.map_err(|e| FetchError::Extract(format!("tar unpack: {e}")))?;
}
}
Ok(())
}
fn url_to_safe_name(url: &str) -> String {
url.chars()
.map(|c| if c.is_alphanumeric() || c == '-' || c == '_' { c } else { '_' })
.collect()
}
fn dirs_cache_dir() -> PathBuf {
if let Some(xdg) = std::env::var_os("XDG_CACHE_HOME")
.map(PathBuf::from)
.filter(|p| p.is_absolute())
{
return xdg;
}
if let Some(home) = std::env::var_os("HOME")
.map(PathBuf::from)
.filter(|p| p.is_absolute())
{
let default = home.join(".cache");
if default.exists() || std::fs::create_dir_all(&default).is_ok() {
return default;
}
}
PathBuf::from("/tmp")
}
#[cfg(test)]
mod archive_report_tests {
use super::*;
fn throttled(retry: Option<u64>) -> FetchError {
FetchError::Throttled { url: "u".into(), status: 429, retry_after: retry }
}
#[test]
fn a_throttle_becomes_a_machine_readable_tag_with_the_servers_advice() {
let f = InputFailure::from_error("nixpkgs", &throttled(Some(120)));
assert_eq!(f.kind, FailureKind::Throttled);
assert_eq!(f.status, Some(429));
assert_eq!(f.retry_after, Some(120));
assert!(f.recoverable_elsewhere);
assert_eq!(f.input, "nixpkgs", "the report must name WHICH input");
}
#[test]
fn each_error_maps_to_its_own_kind() {
let cases: Vec<(FetchError, FailureKind)> = vec![
(throttled(None), FailureKind::Throttled),
(FetchError::Unauthorized { url: "u".into(), status: 403 }, FailureKind::Unauthorized),
(FetchError::NotFound { url: "u".into() }, FailureKind::NotFound),
(FetchError::UnexpectedStatus { url: "u".into(), status: 503 }, FailureKind::UnexpectedStatus),
(FetchError::Download("dns".into()), FailureKind::Transport),
(FetchError::UnsupportedType("hg".into()), FailureKind::Local),
(FetchError::Extract("bad tar".into()), FailureKind::Local),
];
for (err, want) in cases {
let got = InputFailure::from_error("i", &err).kind;
assert_eq!(got, want, "{err:?} classified as {got:?}");
}
}
#[test]
fn only_a_throttle_is_marked_recoverable_elsewhere() {
for e in [
FetchError::Unauthorized { url: "u".into(), status: 401 },
FetchError::NotFound { url: "u".into() },
FetchError::UnexpectedStatus { url: "u".into(), status: 500 },
FetchError::Download("tls".into()),
] {
assert!(
!InputFailure::from_error("i", &e).recoverable_elsewhere,
"{e:?} must not claim another egress would help"
);
}
}
#[test]
fn an_empty_walk_is_NOT_complete() {
let empty = ArchiveReport { scanned: 0, already_present: 0, fetched: 0, failures: vec![] };
assert!(!empty.is_complete(), "an empty walk has not earned 'complete'");
let real = ArchiveReport { scanned: 3, already_present: 3, fetched: 0, failures: vec![] };
assert!(real.is_complete());
}
#[test]
fn recoverable_filters_to_exactly_the_throttles() {
let r = ArchiveReport {
scanned: 4,
already_present: 1,
fetched: 0,
failures: vec![
InputFailure::from_error("a", &throttled(Some(5))),
InputFailure::from_error("b", &FetchError::NotFound { url: "u".into() }),
InputFailure::from_error("c", &throttled(None)),
],
};
let names: Vec<&str> = r.recoverable().map(|f| f.input.as_str()).collect();
assert_eq!(names, vec!["a", "c"]);
assert!(!r.is_complete());
}
#[test]
fn the_json_shape_is_the_contract_a_consumer_reads() {
let r = ArchiveReport {
scanned: 2,
already_present: 1,
fetched: 0,
failures: vec![InputFailure::from_error("nixpkgs", &throttled(Some(90)))],
};
let v: serde_json::Value = serde_json::to_value(&r).expect("serializes");
assert_eq!(v["scanned"], 2);
assert_eq!(v["already_present"], 1);
assert_eq!(v["failures"][0]["kind"], "throttled", "kebab-case tag");
assert_eq!(v["failures"][0]["status"], 429);
assert_eq!(v["failures"][0]["retry_after"], 90);
assert_eq!(v["failures"][0]["recoverable_elsewhere"], true);
assert_eq!(v["failures"][0]["input"], "nixpkgs");
let r2 = ArchiveReport {
scanned: 1,
already_present: 0,
fetched: 0,
failures: vec![InputFailure::from_error("x", &FetchError::Download("dns".into()))],
};
let v2: serde_json::Value = serde_json::to_value(&r2).unwrap();
assert!(v2["failures"][0].get("status").is_none());
assert!(v2["failures"][0].get("retry_after").is_none());
assert_eq!(v2["failures"][0]["kind"], "transport");
}
}
#[cfg(test)]
mod status_classification_tests {
use super::*;
fn resp(status: u16, headers: &[(&str, &str)]) -> ureq::http::Response<()> {
let mut b = ureq::http::Response::builder().status(status);
for (k, v) in headers {
b = b.header(*k, *v);
}
b.body(()).expect("a status + headers response always builds")
}
const URL: &str = "https://api.github.com/repos/o/r/tarball/deadbeef";
#[test]
fn a_429_is_throttled_and_keeps_the_servers_own_retry_after() {
let e = classify_status(URL, &resp(429, &[("retry-after", "120")]));
assert!(matches!(
e,
FetchError::Throttled {
status: 429,
retry_after: Some(120),
..
}
));
assert!(e.is_throttled());
assert_eq!(e.status(), Some(429));
}
#[test]
fn a_429_without_a_header_is_still_throttled() {
let e = classify_status(URL, &resp(429, &[]));
assert!(matches!(e, FetchError::Throttled { retry_after: None, .. }));
assert!(e.is_throttled());
}
#[test]
fn a_403_with_retry_after_is_a_throttle_not_a_credential_fault() {
let e = classify_status(URL, &resp(403, &[("retry-after", "60")]));
assert!(
e.is_throttled(),
"a 403 that says 'come back later' is a throttle, got {e:?}"
);
}
#[test]
fn a_bare_403_is_a_credential_fault_and_NOT_recoverable_elsewhere() {
let e = classify_status(URL, &resp(403, &[]));
assert!(matches!(e, FetchError::Unauthorized { status: 403, .. }));
assert!(!e.is_throttled());
}
#[test]
fn a_404_says_it_may_be_an_invisible_private_input() {
let e = classify_status(URL, &resp(404, &[]));
assert!(matches!(e, FetchError::NotFound { .. }));
assert_eq!(e.status(), Some(404));
let msg = e.to_string();
assert!(msg.contains("invisible to this credential"), "got {msg}");
}
#[test]
fn an_unhandled_status_arrives_as_a_NUMBER_never_as_prose() {
let e = classify_status(URL, &resp(503, &[]));
assert!(matches!(e, FetchError::UnexpectedStatus { status: 503, .. }));
assert_eq!(e.status(), Some(503));
assert!(!e.is_throttled());
}
#[test]
fn no_two_http_failures_render_the_same_bytes() {
let u = URL.to_string();
let cases: Vec<(&str, FetchError)> = vec![
("Throttled(no advice)", FetchError::Throttled { url: u.clone(), status: 404, retry_after: None }),
("Throttled(advice)", FetchError::Throttled { url: u.clone(), status: 404, retry_after: Some(30) }),
("Unauthorized", FetchError::Unauthorized { url: u.clone(), status: 404 }),
("NotFound", FetchError::NotFound { url: u.clone() }),
("UnexpectedStatus", FetchError::UnexpectedStatus { url: u.clone(), status: 404 }),
("Download", FetchError::Download(format!("{u}: connection reset"))),
];
for (i, (name_a, a)) in cases.iter().enumerate() {
for (name_b, b) in cases.iter().skip(i + 1) {
assert_ne!(
a.to_string(),
b.to_string(),
"{name_a} and {name_b} render identically at the same status \
— a caller cannot distinguish them"
);
}
}
}
#[test]
fn an_http_date_retry_after_yields_none_rather_than_a_guess() {
let h = resp(429, &[("retry-after", "Wed, 21 Oct 2026 07:28:00 GMT")]);
assert_eq!(retry_after_seconds(h.headers()), None);
assert!(classify_status(URL, &h).is_throttled());
}
#[test]
fn a_junk_retry_after_does_not_panic_or_lie() {
for v in ["", " ", "abc", "-5", "12.5", "9999999999999999999999"] {
let h = resp(429, &[("retry-after", v)]);
assert_eq!(
retry_after_seconds(h.headers()),
None,
"{v:?} must not parse"
);
}
assert_eq!(retry_after_seconds(resp(429, &[("retry-after", " 30 ")]).headers()), Some(30));
}
#[test]
fn a_transport_failure_is_not_given_a_status() {
let e = FetchError::Download("dns failure".into());
assert_eq!(e.status(), None);
assert!(!e.is_throttled());
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::BTreeMap;
fn make_locked(source_type: &str) -> LockedInput {
LockedInput {
source_type: source_type.to_string(),
owner: None,
repo: None,
rev: None,
nar_hash: None,
last_modified: None,
path: None,
url: None,
git_ref: None,
dir: None,
host: None,
extra: BTreeMap::new(),
}
}
#[test]
fn sanitize_hash_replaces_special_chars() {
assert_eq!(
sanitize_hash("sha256-AAAAAAAAAAAAAAAAAAAAAA="),
"sha256-AAAAAAAAAAAAAAAAAAAAAA"
);
assert_eq!(sanitize_hash("sha256:abc/def="), "sha256-abc_def");
}
#[test]
fn a_traversal_hash_cannot_become_a_path_component() {
for hostile in ["..", ".", "", "../..", "..\u{0}"] {
let s = sanitize_hash(hostile);
assert!(
s != ".." && s != "." && !s.is_empty(),
"{hostile:?} sanitized to {s:?}, still a meaningful component"
);
assert!(
!s.contains('/') && !s.contains('\\'),
"{hostile:?} sanitized to {s:?}, still a separator"
);
}
assert_eq!(sanitize_hash(".."), sanitize_hash(".."));
assert_ne!(sanitize_hash(".."), sanitize_hash("."));
}
#[test]
fn a_well_formed_hash_is_untouched_by_the_guard() {
assert_eq!(
sanitize_hash("sha256-avzRM+ffKgikqMRcOhhYp3ifgwXMGbH0rEGEZPEGMYE="),
"sha256-avzRM+ffKgikqMRcOhhYp3ifgwXMGbH0rEGEZPEGMYE"
);
assert_eq!(sanitize_hash("sha256:abc/def="), "sha256-abc_def");
}
#[test]
fn publishing_an_immutable_tree_adopts_the_winner_and_deletes_nothing() {
let tmp = tempfile::tempdir().unwrap();
let dest = tmp.path().join("github-o-r-deadbeef");
let staging = staging_path(&dest);
std::fs::create_dir_all(&dest).unwrap();
std::fs::write(dest.join("theirs"), b"x").unwrap();
std::fs::create_dir_all(&staging).unwrap();
std::fs::write(staging.join("ours"), b"y").unwrap();
publish(&staging, &dest, true).unwrap();
assert!(
dest.join("theirs").exists(),
"an immutable tree is content-addressed: the winner's tree IS ours, \
and deleting it to install an identical one is pure risk"
);
assert!(!staging.exists(), "our staging must be cleaned up");
}
#[test]
fn publishing_a_mutable_tree_replaces_it_without_a_delete_in_place() {
let tmp = tempfile::tempdir().unwrap();
let dest = tmp.path().join("github-o-r-main");
let staging = staging_path(&dest);
std::fs::create_dir_all(&dest).unwrap();
std::fs::write(dest.join("old"), b"x").unwrap();
std::fs::create_dir_all(&staging).unwrap();
std::fs::write(staging.join("new"), b"y").unwrap();
publish(&staging, &dest, false).unwrap();
assert!(dest.join("new").exists(), "the new tree must be published");
assert!(!dest.join("old").exists(), "and must REPLACE, not union");
assert!(!staging.exists());
let leftovers: Vec<_> = std::fs::read_dir(tmp.path())
.unwrap()
.filter_map(Result::ok)
.map(|e| e.file_name().to_string_lossy().into_owned())
.filter(|n| n.contains(".old"))
.collect();
assert!(leftovers.is_empty(), "aside dirs left behind: {leftovers:?}");
}
#[test]
fn staging_is_scoped_by_thread_not_only_by_pid() {
let dest = std::path::Path::new("/c/inputs/github-o-r-deadbeef");
let here = staging_path(dest);
let there = std::thread::spawn(move || staging_path(dest))
.join()
.unwrap();
assert_ne!(
here, there,
"two threads must not share a staging directory"
);
}
#[test]
fn only_a_full_object_id_is_treated_as_immutable() {
assert!(is_immutable_rev("7fd33221240a3ab97781a066c5efe0124979527f"));
assert!(is_immutable_rev(&"a".repeat(64)));
assert!(!is_immutable_rev("main"), "a branch name is not a commit");
assert!(!is_immutable_rev("v1.2.3"), "a tag can be moved");
assert!(!is_immutable_rev("7fd3322"), "a short rev is ambiguous");
assert!(!is_immutable_rev(""), "an empty rev names nothing");
assert!(!is_immutable_rev(&"z".repeat(40)));
assert!(!is_immutable_rev(&"A".repeat(40)));
}
#[test]
fn staging_is_a_sibling_so_the_publish_rename_is_atomic() {
let dest = std::path::Path::new("/cache/sui/inputs/sha256-abc/github-o-r-deadbeef");
let staging = staging_path(dest);
assert_eq!(
staging.parent(),
dest.parent(),
"staging in /tmp would put the rename across filesystems, where it \
is a copy — and a copy is not atomic, which is the whole point"
);
assert_ne!(staging, dest.to_path_buf());
let name = staging.file_name().unwrap().to_string_lossy().into_owned();
assert!(name.starts_with('.'), "hidden, so it is not mistaken for a tree");
assert!(
name.contains(&std::process::id().to_string()),
"pid-scoped, so two concurrent fetchers cannot share a staging dir"
);
let dotted = std::path::Path::new("/c/github-o-r-1.2.3");
assert!(
staging_path(dotted)
.file_name()
.unwrap()
.to_string_lossy()
.contains("github-o-r-1.2.3"),
"the full directory name must survive into the staging name"
);
}
#[test]
fn find_single_subdir_returns_child_when_one_dir() {
let tmp = tempfile::tempdir().unwrap();
let child = tmp.path().join("repo-abc123");
std::fs::create_dir(&child).unwrap();
std::fs::write(child.join("file.txt"), "hello").unwrap();
let result = find_single_subdir_or_self(tmp.path());
assert_eq!(result, child);
}
#[test]
fn find_single_subdir_returns_self_when_multiple() {
let tmp = tempfile::tempdir().unwrap();
std::fs::create_dir(tmp.path().join("a")).unwrap();
std::fs::create_dir(tmp.path().join("b")).unwrap();
let result = find_single_subdir_or_self(tmp.path());
assert_eq!(result, tmp.path());
}
#[test]
fn find_single_subdir_returns_self_when_empty() {
let tmp = tempfile::tempdir().unwrap();
let result = find_single_subdir_or_self(tmp.path());
assert_eq!(result, tmp.path());
}
#[test]
fn find_single_subdir_returns_self_when_child_is_file() {
let tmp = tempfile::tempdir().unwrap();
std::fs::write(tmp.path().join("file.txt"), "data").unwrap();
let result = find_single_subdir_or_self(tmp.path());
assert_eq!(result, tmp.path());
}
#[test]
fn url_to_safe_name_replaces_slashes_and_colons() {
let name = url_to_safe_name("https://example.com/foo/bar.tar.gz");
assert!(!name.contains('/'));
assert!(!name.contains(':'));
assert!(name.contains("example"));
}
#[test]
fn fetcher_with_custom_cache_dir() {
let tmp = tempfile::tempdir().unwrap();
let fetcher = InputFetcher::with_cache_dir(tmp.path().to_path_buf());
assert_eq!(fetcher.cache_dir(), tmp.path());
}
#[test]
fn fetcher_default_cache_dir_exists() {
let fetcher = InputFetcher::new();
let path_str = fetcher.cache_dir().to_string_lossy();
assert!(path_str.ends_with("sui/inputs"), "got: {path_str}");
}
#[test]
fn fetch_path_returns_filesystem_path() {
let tmp = tempfile::tempdir().unwrap();
let fetcher = InputFetcher::with_cache_dir(tmp.path().join("cache"));
let mut locked = make_locked("path");
locked.path = Some("/var/empty/dep".to_string());
let result = fetcher.fetch(&locked).unwrap();
assert_eq!(result, PathBuf::from("/var/empty/dep"));
}
#[test]
fn fetch_path_missing_field_errors() {
let tmp = tempfile::tempdir().unwrap();
let fetcher = InputFetcher::with_cache_dir(tmp.path().join("cache"));
let locked = make_locked("path");
let result = fetcher.fetch(&locked);
assert!(result.is_err());
assert!(result.unwrap_err().to_string().contains("path"));
}
#[test]
fn fetch_unsupported_type_returns_error() {
let tmp = tempfile::tempdir().unwrap();
let fetcher = InputFetcher::with_cache_dir(tmp.path().join("cache"));
let locked = make_locked("mercurial");
let result = fetcher.fetch(&locked);
assert!(matches!(result, Err(FetchError::UnsupportedType(_))));
}
#[test]
fn gitlab_archive_url_is_well_formed() {
assert_eq!(
InputFetcher::gitlab_archive_url(None, "group", "proj", "abc123"),
"https://gitlab.com/group/proj/-/archive/abc123/proj-abc123.tar.gz"
);
}
#[test]
fn gitlab_archive_url_honors_custom_host() {
assert_eq!(
InputFetcher::gitlab_archive_url(Some("gitlab.gnome.org"), "GNOME", "gnome-shell", "abc"),
"https://gitlab.gnome.org/GNOME/gnome-shell/-/archive/abc/gnome-shell-abc.tar.gz"
);
}
#[test]
fn sourcehut_archive_url_prepends_tilde() {
assert_eq!(
InputFetcher::sourcehut_archive_url("emersion", "page", "HEAD"),
"https://git.sr.ht/~emersion/page/archive/HEAD.tar.gz"
);
assert_eq!(
InputFetcher::sourcehut_archive_url("~emersion", "page", "HEAD"),
"https://git.sr.ht/~emersion/page/archive/HEAD.tar.gz"
);
}
#[test]
fn cache_hit_returns_cached_path() {
let tmp = tempfile::tempdir().unwrap();
let cache_dir = tmp.path().join("cache");
std::fs::create_dir_all(&cache_dir).unwrap();
let hash = "sha256-TESTCACHEHIT";
let cached_dir = cache_dir.join(sanitize_hash(hash));
std::fs::create_dir_all(&cached_dir).unwrap();
std::fs::write(cached_dir.join("flake.nix"), "{}").unwrap();
let fetcher = InputFetcher::with_cache_dir(cache_dir);
let mut locked = make_locked("github");
locked.nar_hash = Some(hash.to_string());
let result = fetcher.fetch(&locked).unwrap();
assert_eq!(result, cached_dir);
}
#[test]
fn github_archive_url_format() {
let url = InputFetcher::github_archive_url("nixos", "nixpkgs", "abc123");
assert_eq!(
url,
"https://github.com/nixos/nixpkgs/archive/abc123.tar.gz"
);
}
#[test]
fn fetch_github_missing_owner_errors() {
let tmp = tempfile::tempdir().unwrap();
let fetcher = InputFetcher::with_cache_dir(tmp.path().join("cache"));
let mut locked = make_locked("github");
locked.repo = Some("nixpkgs".into());
locked.rev = Some("abc123".into());
let result = fetcher.fetch(&locked);
assert!(result.is_err());
assert!(result.unwrap_err().to_string().contains("owner"));
}
#[test]
fn fetch_github_missing_rev_errors() {
let tmp = tempfile::tempdir().unwrap();
let fetcher = InputFetcher::with_cache_dir(tmp.path().join("cache"));
let mut locked = make_locked("github");
locked.owner = Some("nixos".into());
locked.repo = Some("nixpkgs".into());
let result = fetcher.fetch(&locked);
assert!(result.is_err());
assert!(result.unwrap_err().to_string().contains("rev"));
}
#[test]
fn fetch_git_missing_url_errors() {
let tmp = tempfile::tempdir().unwrap();
let fetcher = InputFetcher::with_cache_dir(tmp.path().join("cache"));
let mut locked = make_locked("git");
locked.rev = Some("abc123".into());
let result = fetcher.fetch(&locked);
assert!(result.is_err());
assert!(result.unwrap_err().to_string().contains("url"));
}
#[test]
fn fetch_git_missing_rev_errors() {
let tmp = tempfile::tempdir().unwrap();
let fetcher = InputFetcher::with_cache_dir(tmp.path().join("cache"));
let mut locked = make_locked("git");
locked.url = Some("https://example.com/repo.git".into());
let result = fetcher.fetch(&locked);
assert!(result.is_err());
assert!(result.unwrap_err().to_string().contains("rev"));
}
#[test]
fn fetch_tarball_missing_url_errors() {
let tmp = tempfile::tempdir().unwrap();
let fetcher = InputFetcher::with_cache_dir(tmp.path().join("cache"));
let locked = make_locked("tarball");
let result = fetcher.fetch(&locked);
assert!(result.is_err());
assert!(result.unwrap_err().to_string().contains("url"));
}
#[test]
fn extract_tar_gz_empty_archive_errors() {
let tmp = tempfile::tempdir().unwrap();
let result = extract_tar_gz(&[], tmp.path());
assert!(result.is_err());
}
#[test]
fn extract_tar_gz_invalid_data_errors() {
let tmp = tempfile::tempdir().unwrap();
let result = extract_tar_gz(b"not a gzip stream at all", tmp.path());
assert!(result.is_err());
}
#[test]
fn dest_dir_uses_nar_hash_when_present() {
let fetcher = InputFetcher::with_cache_dir(PathBuf::from("/cache"));
let mut locked = make_locked("github");
locked.nar_hash = Some("sha256-ABC123=".to_string());
let dest = fetcher.dest_dir(&locked, "fallback");
assert!(dest.to_string_lossy().contains("sha256-ABC123"));
assert!(!dest.to_string_lossy().contains("fallback"));
}
#[test]
fn dest_dir_uses_fallback_when_no_hash() {
let fetcher = InputFetcher::with_cache_dir(PathBuf::from("/cache"));
let locked = make_locked("github");
let dest = fetcher.dest_dir(&locked, "fallback-name");
assert!(dest.to_string_lossy().contains("fallback-name"));
}
#[test]
fn is_non_empty_dir_returns_true_for_non_empty() {
let tmp = tempfile::tempdir().unwrap();
std::fs::write(tmp.path().join("file.txt"), "data").unwrap();
assert!(is_non_empty_dir(tmp.path()));
}
#[test]
fn is_non_empty_dir_returns_false_for_empty() {
let tmp = tempfile::tempdir().unwrap();
assert!(!is_non_empty_dir(tmp.path()));
}
#[test]
fn is_non_empty_dir_returns_false_for_missing() {
assert!(!is_non_empty_dir(Path::new("/nonexistent/path/12345")));
}
#[test]
fn empty_cache_dir_is_treated_as_miss() {
let tmp = tempfile::tempdir().unwrap();
let cache_dir = tmp.path().join("cache");
std::fs::create_dir_all(&cache_dir).unwrap();
let hash = "sha256-EMPTYTEST";
let cached_dir = cache_dir.join(sanitize_hash(hash));
std::fs::create_dir_all(&cached_dir).unwrap();
assert!(std::fs::read_dir(&cached_dir).unwrap().next().is_none());
let fetcher = InputFetcher::with_cache_dir(cache_dir);
let mut locked = make_locked("github");
locked.nar_hash = Some(hash.to_string());
let result = fetcher.fetch(&locked);
assert!(result.is_err(), "should not return stale empty cache");
assert!(!cached_dir.exists(), "stale cache dir should be removed");
}
#[test]
fn tarball_from_https_github() {
let url = github_tarball_from_git_url(
"https://github.com/NixOS/nixpkgs.git",
"abc123",
);
assert_eq!(
url.as_deref(),
Some("https://github.com/NixOS/nixpkgs/archive/abc123.tar.gz")
);
}
#[test]
fn tarball_from_git_plus_https() {
let url = github_tarball_from_git_url(
"git+https://github.com/NixOS/nixpkgs",
"def456",
);
assert_eq!(
url.as_deref(),
Some("https://github.com/NixOS/nixpkgs/archive/def456.tar.gz")
);
}
#[test]
fn tarball_from_non_github_returns_none() {
assert!(github_tarball_from_git_url("https://gitlab.com/foo/bar.git", "abc").is_none());
assert!(github_tarball_from_git_url("ssh://git@github.com/foo/bar", "abc").is_none());
}
#[test]
fn tarball_from_malformed_path_returns_none() {
assert!(github_tarball_from_git_url("https://github.com/", "abc").is_none());
assert!(github_tarball_from_git_url("https://github.com/only-owner", "abc").is_none());
}
}
pub fn resolve_flake_dir(flake_ref: &FlakeRef) -> Result<std::path::PathBuf, FetchError> {
match flake_ref.local_dir() {
Some(p) => Ok(p.to_path_buf()),
None => {
let locked = flake_ref
.source
.locked_input()
.ok_or_else(|| FetchError::UnsupportedType("non-fetchable flake source".into()))?;
InputFetcher::new().fetch(&locked)
}
}
}