use serde::{Deserialize, Serialize};
use tatara_lisp::DeriveTataraDomain;
use crate::SpecError;
#[derive(DeriveTataraDomain, Serialize, Deserialize, Debug, Clone)]
#[tatara(keyword = "deffetcher")]
pub struct FetcherSpec {
pub name: String,
pub transport: FetchTransport,
#[serde(rename = "hashMode")]
pub hash_mode: FetchHashMode,
#[serde(rename = "outputKind")]
pub output_kind: FetcherOutputKind,
pub phases: Vec<FetcherPhase>,
}
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum FetchTransport {
Http,
Git,
Tree,
LocalPath,
Mercurial,
}
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
pub enum FetchHashMode {
Flat,
Recursive,
Sri,
}
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
pub enum FetcherOutputKind {
FixedOutput,
ContentAddressed,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct FetcherPhase {
pub kind: FetcherPhaseKind,
#[serde(default)]
pub bind: Option<String>,
#[serde(default)]
pub from: Option<String>,
}
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
pub enum FetcherPhaseKind {
ValidateUrl,
ResolveRegistryRef,
FetchBytes,
Unpack,
CheckHash,
WriteToStore,
CacheLookup,
EmitNarHash,
}
pub struct FetchArgs {
pub url: String,
pub declared_hash: Option<String>,
pub name_hint: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FetchOutcome {
pub store_path: String,
pub nar_hash: String,
}
pub trait FetcherEnvironment {
fn fetch_bytes(&self, url: &str) -> Result<Vec<u8>, String>;
fn hash_bytes(&self, bytes: &[u8]) -> String;
fn write_to_store(&self, name: &str, bytes: &[u8]) -> Result<String, String>;
fn cache_lookup(&self, _name: &str, _declared_hash: &str) -> Result<Option<String>, String> {
Ok(None)
}
}
pub trait HttpTransport {
fn get(&self, url: &str) -> Result<Vec<u8>, HttpError>;
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum HttpError {
BadUrl(String),
UnsupportedScheme(String),
NetworkFailure(String),
NotFound(String),
Forbidden(String),
BodyReadFailure(String),
IoError(String),
}
impl std::fmt::Display for HttpError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::BadUrl(m) => write!(f, "bad URL: {m}"),
Self::UnsupportedScheme(s) => write!(f, "unsupported scheme `{s}`"),
Self::NetworkFailure(m) => write!(f, "network: {m}"),
Self::NotFound(m) => write!(f, "not found: {m}"),
Self::Forbidden(m) => write!(f, "forbidden: {m}"),
Self::BodyReadFailure(m) => write!(f, "body read: {m}"),
Self::IoError(m) => write!(f, "io: {m}"),
}
}
}
impl std::error::Error for HttpError {}
pub struct FsTransport;
impl HttpTransport for FsTransport {
fn get(&self, url: &str) -> Result<Vec<u8>, HttpError> {
let parsed = url::Url::parse(url).map_err(|e| HttpError::BadUrl(e.to_string()))?;
if parsed.scheme() != "file" {
return Err(HttpError::UnsupportedScheme(parsed.scheme().to_string()));
}
let path = parsed.to_file_path()
.map_err(|_| HttpError::BadUrl(format!("non-file URL `{url}`")))?;
std::fs::read(&path).map_err(|e| match e.kind() {
std::io::ErrorKind::NotFound => HttpError::NotFound(path.display().to_string()),
std::io::ErrorKind::PermissionDenied => HttpError::Forbidden(path.display().to_string()),
_ => HttpError::IoError(e.to_string()),
})
}
}
#[derive(Default)]
pub struct MockTransport {
pub responses: std::collections::HashMap<String, Vec<u8>>,
}
impl MockTransport {
pub fn with(mut self, url: &str, bytes: Vec<u8>) -> Self {
self.responses.insert(url.to_string(), bytes);
self
}
}
impl HttpTransport for MockTransport {
fn get(&self, url: &str) -> Result<Vec<u8>, HttpError> {
self.responses.get(url).cloned()
.ok_or_else(|| HttpError::NotFound(url.to_string()))
}
}
pub struct SchemeRouter<R: HttpTransport> {
pub remote: R,
}
impl<R: HttpTransport> SchemeRouter<R> {
pub fn new(remote: R) -> Self { Self { remote } }
}
impl<R: HttpTransport> HttpTransport for SchemeRouter<R> {
fn get(&self, url: &str) -> Result<Vec<u8>, HttpError> {
let parsed = url::Url::parse(url).map_err(|e| HttpError::BadUrl(e.to_string()))?;
match parsed.scheme() {
"file" => FsTransport.get(url),
"http" | "https" => self.remote.get(url),
other => Err(HttpError::UnsupportedScheme(other.to_string())),
}
}
}
pub fn apply<E: FetcherEnvironment>(
spec: &FetcherSpec,
args: &FetchArgs,
env: &E,
) -> Result<FetchOutcome, SpecError> {
if spec.transport != FetchTransport::Http
|| spec.hash_mode != FetchHashMode::Flat
{
return Err(SpecError::Interp {
phase: "fetcher-unimplemented".into(),
message: format!(
"fetcher `{}` (transport {:?}, hash-mode {:?}) — \
M3.0 supports only Http+Flat (fetchurl). M3.1+ \
implementations land per-transport.",
spec.name, spec.transport, spec.hash_mode,
),
});
}
let name = args.name_hint.as_deref().unwrap_or("download");
for phase in &spec.phases {
match phase.kind {
FetcherPhaseKind::ValidateUrl => validate_url(&args.url)?,
FetcherPhaseKind::ResolveRegistryRef => {
}
FetcherPhaseKind::CacheLookup => {
if let Some(declared) = args.declared_hash.as_deref() {
let hit = env
.cache_lookup(name, declared)
.map_err(|e| SpecError::Interp {
phase: "cache-lookup".into(),
message: e,
})?;
if let Some(path) = hit {
return Ok(FetchOutcome {
store_path: path,
nar_hash: declared.to_string(),
});
}
}
}
FetcherPhaseKind::FetchBytes => {
}
FetcherPhaseKind::Unpack => {
}
FetcherPhaseKind::CheckHash | FetcherPhaseKind::WriteToStore
| FetcherPhaseKind::EmitNarHash => {
}
}
}
let bytes = env.fetch_bytes(&args.url).map_err(|e| SpecError::Interp {
phase: "fetch-bytes".into(),
message: format!("fetching `{}`: {e}", args.url),
})?;
let computed = env.hash_bytes(&bytes);
if let Some(declared) = args.declared_hash.as_deref() {
if declared != computed {
return Err(SpecError::Interp {
phase: "hash-mismatch".into(),
message: format!(
"hash mismatch for `{}`: declared {declared}, got {computed}",
args.url,
),
});
}
}
let store_path = env
.write_to_store(name, &bytes)
.map_err(|e| SpecError::Interp {
phase: "write-to-store".into(),
message: format!("writing `{name}`: {e}"),
})?;
Ok(FetchOutcome { store_path, nar_hash: computed })
}
fn validate_url(url: &str) -> Result<(), SpecError> {
if url.is_empty() {
return Err(SpecError::Interp {
phase: "url-validate".into(),
message: "url is empty".into(),
});
}
let allowed = ["http://", "https://", "file://"];
if !allowed.iter().any(|p| url.starts_with(p)) {
return Err(SpecError::Interp {
phase: "url-validate".into(),
message: format!(
"url `{url}` uses an unsupported scheme \
(allowed: http://, https://, file://)",
),
});
}
Ok(())
}
pub const CANONICAL_FETCHERS_LISP: &str = include_str!("../specs/fetchers.lisp");
pub fn load_canonical() -> Result<Vec<FetcherSpec>, SpecError> {
crate::loader::load_all::<FetcherSpec>(CANONICAL_FETCHERS_LISP)
}
pub fn load_named(name: &str) -> Result<FetcherSpec, SpecError> {
load_canonical()?
.into_iter()
.find(|f| f.name == name)
.ok_or_else(|| SpecError::Load(format!("no (deffetcher) with :name {name:?}")))
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashSet;
#[test]
fn canonical_fetchers_parse() {
let specs = load_canonical().expect("canonical fetchers must compile");
assert!(!specs.is_empty());
}
#[test]
fn every_cppnix_fetcher_named() {
let specs = load_canonical().unwrap();
let names: HashSet<&str> = specs.iter().map(|f| f.name.as_str()).collect();
for required in ["fetchurl", "fetchTarball", "fetchGit", "fetchTree", "path"] {
assert!(
names.contains(required),
"canonical fetcher corpus missing `{required}`",
);
}
}
#[test]
fn fetchurl_uses_http_flat() {
let f = load_named("fetchurl").unwrap();
assert_eq!(f.transport, FetchTransport::Http);
assert_eq!(f.hash_mode, FetchHashMode::Flat);
assert_eq!(f.output_kind, FetcherOutputKind::FixedOutput);
}
#[test]
fn fetchgit_uses_git_recursive() {
let f = load_named("fetchGit").unwrap();
assert_eq!(f.transport, FetchTransport::Git);
assert_eq!(f.hash_mode, FetchHashMode::Recursive);
}
#[test]
fn every_fetcher_has_validate_and_writetostore() {
let specs = load_canonical().unwrap();
for spec in &specs {
let kinds: Vec<FetcherPhaseKind> =
spec.phases.iter().map(|p| p.kind).collect();
assert!(
kinds.contains(&FetcherPhaseKind::ValidateUrl)
|| spec.transport == FetchTransport::LocalPath,
"{}: every network fetcher must ValidateUrl",
spec.name,
);
assert!(
kinds.contains(&FetcherPhaseKind::WriteToStore),
"{}: missing WriteToStore",
spec.name,
);
}
}
use std::cell::RefCell;
use std::collections::HashMap;
struct MockEnv {
responses: HashMap<String, Vec<u8>>,
store: RefCell<HashMap<String, Vec<u8>>>,
}
impl MockEnv {
fn new() -> Self {
Self {
responses: HashMap::new(),
store: RefCell::new(HashMap::new()),
}
}
fn with_response(mut self, url: &str, body: &[u8]) -> Self {
self.responses.insert(url.into(), body.to_vec());
self
}
}
impl FetcherEnvironment for MockEnv {
fn fetch_bytes(&self, url: &str) -> Result<Vec<u8>, String> {
self.responses
.get(url)
.cloned()
.ok_or_else(|| format!("no canned response for {url}"))
}
fn hash_bytes(&self, bytes: &[u8]) -> String {
let first = bytes.first().copied().unwrap_or(0);
format!("sha256:test-{}-{:02x}", bytes.len(), first)
}
fn write_to_store(&self, name: &str, bytes: &[u8]) -> Result<String, String> {
let path = format!("/nix/store/abc-{name}");
self.store.borrow_mut().insert(path.clone(), bytes.to_vec());
Ok(path)
}
}
#[test]
fn fetchurl_happy_path() {
let spec = load_named("fetchurl").unwrap();
let env = MockEnv::new()
.with_response("https://example.com/hello.tar", b"hello\n");
let args = FetchArgs {
url: "https://example.com/hello.tar".into(),
declared_hash: None,
name_hint: Some("hello.tar".into()),
};
let outcome = apply(&spec, &args, &env).unwrap();
assert_eq!(outcome.store_path, "/nix/store/abc-hello.tar");
assert!(outcome.nar_hash.starts_with("sha256:"));
assert_eq!(
env.store.borrow().get("/nix/store/abc-hello.tar"),
Some(&b"hello\n".to_vec()),
);
}
#[test]
fn fetchurl_rejects_malformed_url() {
let spec = load_named("fetchurl").unwrap();
let env = MockEnv::new();
let args = FetchArgs {
url: "ftp://example.com/x".into(),
declared_hash: None,
name_hint: None,
};
let err = apply(&spec, &args, &env).unwrap_err();
match err {
SpecError::Interp { phase, .. } => assert_eq!(phase, "url-validate"),
_ => panic!("expected url-validate error"),
}
}
#[test]
fn fetchurl_rejects_empty_url() {
let spec = load_named("fetchurl").unwrap();
let env = MockEnv::new();
let args = FetchArgs {
url: String::new(),
declared_hash: None,
name_hint: None,
};
let err = apply(&spec, &args, &env).unwrap_err();
match err {
SpecError::Interp { phase, .. } => assert_eq!(phase, "url-validate"),
_ => panic!("expected url-validate"),
}
}
#[test]
fn fetchurl_verifies_declared_hash() {
let spec = load_named("fetchurl").unwrap();
let env = MockEnv::new()
.with_response("https://example.com/x", b"hello");
let args = FetchArgs {
url: "https://example.com/x".into(),
declared_hash: Some("sha256:fake-hash".into()),
name_hint: Some("x".into()),
};
let err = apply(&spec, &args, &env).unwrap_err();
match err {
SpecError::Interp { phase, message } => {
assert_eq!(phase, "hash-mismatch");
assert!(message.contains("fake-hash"));
}
_ => panic!("expected hash-mismatch"),
}
}
#[test]
fn fetchurl_accepts_matching_hash() {
let spec = load_named("fetchurl").unwrap();
let body = b"hello";
let env = MockEnv::new().with_response("https://example.com/x", body);
let expected = env.hash_bytes(body);
let args = FetchArgs {
url: "https://example.com/x".into(),
declared_hash: Some(expected.clone()),
name_hint: Some("x".into()),
};
let outcome = apply(&spec, &args, &env).unwrap();
assert_eq!(outcome.nar_hash, expected);
}
#[test]
fn cache_hit_short_circuits_fetch() {
struct CacheHitEnv;
impl FetcherEnvironment for CacheHitEnv {
fn fetch_bytes(&self, _: &str) -> Result<Vec<u8>, String> {
Err("fetch should NOT have been called on cache hit".into())
}
fn hash_bytes(&self, _: &[u8]) -> String { unreachable!() }
fn write_to_store(&self, _: &str, _: &[u8]) -> Result<String, String> {
unreachable!()
}
fn cache_lookup(&self, _: &str, h: &str) -> Result<Option<String>, String> {
Ok(Some(format!("/nix/store/cached-{h}")))
}
}
let spec = load_named("fetchurl").unwrap();
let args = FetchArgs {
url: "https://example.com/x".into(),
declared_hash: Some("sha256:abc".into()),
name_hint: Some("x".into()),
};
let outcome = apply(&spec, &args, &CacheHitEnv).unwrap();
assert_eq!(outcome.store_path, "/nix/store/cached-sha256:abc");
}
#[test]
fn non_fetchurl_transport_returns_typed_not_yet() {
let spec = load_named("fetchGit").unwrap();
let env = MockEnv::new();
let args = FetchArgs {
url: "https://example.com/repo.git".into(),
declared_hash: None,
name_hint: Some("repo".into()),
};
let err = apply(&spec, &args, &env).unwrap_err();
match err {
SpecError::Interp { phase, message } => {
assert_eq!(phase, "fetcher-unimplemented");
assert!(message.contains("Git"));
}
_ => panic!("expected fetcher-unimplemented"),
}
}
}