use std::{
env,
ffi::OsStr,
fs,
path::{Component, Path, PathBuf},
process::Command,
};
use anyhow::{bail, Context, Result};
use super::{checked, download_https, managed_bin_dir, Adapter};
use crate::members::{Kind, Manifest, Program, ANY_TARGET};
use crate::model::ToolName;
pub(super) struct ManifestAdapter(pub(super) &'static Manifest);
pub(crate) fn release_target(manifest: &Manifest) -> Result<&'static str> {
if manifest.kind == Kind::PythonWheel {
return Ok(ANY_TARGET);
}
let host = match (env::consts::ARCH, env::consts::OS) {
("x86_64", "linux") => "x86_64-unknown-linux-gnu",
("aarch64", "linux") => "aarch64-unknown-linux-gnu",
("x86_64", "macos") => "x86_64-apple-darwin",
("aarch64", "macos") => "aarch64-apple-darwin",
(arch, os) => bail!("{}: no release target for {arch}-{os}", manifest.name),
};
if !manifest.sha256.contains_key(host) {
bail!(
"{} {} has no reviewed digest for {host}; this host cannot install it",
manifest.name,
manifest.version
);
}
Ok(host)
}
fn destination(program: &str) -> Result<PathBuf> {
Ok(managed_bin_dir()
.context("HOME is required for a stack member")?
.join(program))
}
fn on_path(program: &str) -> Option<PathBuf> {
use std::os::unix::fs::PermissionsExt;
env::split_paths(&env::var_os("PATH")?)
.map(|dir| dir.join(program))
.find(|p| {
p.is_file()
&& p.metadata()
.is_ok_and(|m| m.permissions().mode() & 0o111 != 0)
})
}
fn same_file(a: &Path, b: &Path) -> bool {
match (a.canonicalize(), b.canonicalize()) {
(Ok(a), Ok(b)) => a == b,
_ => false,
}
}
fn identify(path: &Path, program: &Program) -> Result<()> {
identify_via(path, program, None)
}
fn identify_via(path: &Path, program: &Program, wrapper: Option<&Path>) -> Result<()> {
let root = tempfile::tempdir().context("create identity probe root")?;
let out = hermetic_command(root.path(), wrapper, path)?
.args(&program.identity_argv)
.output()
.with_context(|| format!("cannot run {} to read its identity", path.display()))?;
let text = format!(
"{}{}",
String::from_utf8_lossy(&out.stdout),
String::from_utf8_lossy(&out.stderr)
);
if !text.contains(&program.identity_contains) {
bail!(
"a different `{}` is at {} (its identity does not contain {:?})",
program.name,
path.display(),
program.identity_contains
);
}
Ok(())
}
fn refuse_foreign_program(manifest: &Manifest) -> Result<()> {
for program in &manifest.programs {
let dest = destination(&program.name)?;
if dest.exists() {
if let Err(error) = identify(&dest, program) {
match orphaned_own_launcher(manifest, &dest) {
Some(interpreter) => eprintln!(
"{}: replacing {}: it is this member's own console script, and its \
venv interpreter {} no longer exists",
manifest.name,
dest.display(),
interpreter.display()
),
None => {
return Err(error).with_context(|| {
format!("refusing to overwrite it with {}", manifest.name)
})
}
}
}
}
if let Some(found) = on_path(&program.name).filter(|p| !same_file(p, &dest)) {
identify(&found, program).with_context(|| {
format!(
"it would shadow {} installed at {}",
manifest.name,
dest.display()
)
})?;
}
}
Ok(())
}
fn orphaned_own_launcher(manifest: &Manifest, script: &Path) -> Option<PathBuf> {
if manifest.kind != Kind::PythonWheel {
return None;
}
let owned = venv_root(manifest).ok()?.parent()?.to_path_buf();
let text = fs::read(script).ok()?;
let text = String::from_utf8_lossy(&text[..text.len().min(4096)]);
let mut lines = text.lines();
let shebang = lines.next()?.strip_prefix("#!")?.trim();
let interpreter = if shebang == "/bin/sh" {
lines
.next()?
.strip_prefix("'''exec' ")?
.split_whitespace()
.next()?
.to_string()
} else {
shebang.split_whitespace().next()?.to_string()
};
let interpreter = PathBuf::from(interpreter);
let plain = interpreter.is_absolute()
&& interpreter
.components()
.all(|c| matches!(c, Component::RootDir | Component::Normal(_)));
let inside = plain
&& interpreter
.strip_prefix(&owned)
.is_ok_and(|rest| !rest.as_os_str().is_empty());
(inside && !interpreter.exists()).then_some(interpreter)
}
fn resolve(program: &Program) -> Result<PathBuf> {
let dest = destination(&program.name)?;
let path = if dest.exists() {
dest
} else {
on_path(&program.name).with_context(|| format!("`{}` is not installed", program.name))?
};
identify(&path, program)?;
Ok(path)
}
fn install_atomically(source: &Path, dest: &Path) -> Result<()> {
use std::os::unix::fs::PermissionsExt;
let dir = dest.parent().context("destination has no directory")?;
let staged = tempfile::NamedTempFile::new_in(dir)
.with_context(|| format!("stage in {}", dir.display()))?;
fs::copy(source, staged.path()).with_context(|| format!("stage {}", dest.display()))?;
fs::set_permissions(staged.path(), fs::Permissions::from_mode(0o755))?;
staged.as_file().sync_all()?;
staged
.persist(dest)
.map_err(|e| e.error)
.with_context(|| format!("install {}", dest.display()))?;
Ok(())
}
pub(super) fn sha256_file(path: &Path) -> Result<String> {
use sha2::{Digest, Sha256};
let mut file = fs::File::open(path).with_context(|| format!("open {}", path.display()))?;
let mut hasher = Sha256::new();
std::io::copy(&mut file, &mut hasher)?;
Ok(format!("{:x}", hasher.finalize()))
}
pub(crate) fn stage(m: &Manifest) -> Result<(tempfile::TempDir, PathBuf)> {
let target = release_target(m)?;
let asset = m.asset_for(target);
let root =
tempfile::tempdir().with_context(|| format!("create {} download directory", m.name))?;
let archive = root.path().join(&asset);
download_https(
&format!(
"https://github.com/{}/releases/download/{}/{asset}",
m.repo,
m.tag()
),
&archive,
)?;
if sha256_file(&archive)? != m.sha256[target] {
bail!(
"{} {} release checksum mismatch for {target}",
m.name,
m.version
);
}
let unpack = match m.kind {
Kind::RustRelease => root.path().join("unpack"),
Kind::PythonWheel => venv_root(m)?,
};
let unpack = materialize(m, &archive, &unpack)?;
Ok((root, unpack))
}
pub(crate) fn verify_release_member(m: &Manifest) -> Result<()> {
if m.kind != Kind::RustRelease {
bail!("standalone release verification supports rust-release members only");
}
let (_root, unpack) = stage(m)?;
verify_staged_member(m, &unpack)
}
fn verify_staged_member(m: &Manifest, unpack: &Path) -> Result<()> {
let resolved = m
.programs
.iter()
.map(|program| {
let path = find_program(unpack, &program.name).context("release program missing")?;
identify(&path, program)?;
Ok((program.name.as_str(), path))
})
.collect::<Result<Vec<_>>>()?;
let first = &resolved[0].1;
let root = tempfile::tempdir()?;
let mut argv = vec![first.to_string_lossy().into_owned()];
argv.extend(m.version_argv.iter().cloned());
let output = hermetic_via(root.path(), None, &argv, None)?;
let reported = String::from_utf8_lossy(&output.stdout);
if !reports_version(reported.trim(), &m.version) {
bail!("{} release reports an unexpected version", m.name);
}
verify_resolved(m, &resolved)
}
impl Adapter for ManifestAdapter {
fn name(&self) -> ToolName {
ToolName::Member(self.0.name.as_str())
}
fn desired_version(&self) -> String {
format!("{} {}", self.0.name, self.0.version)
}
fn package_adapter(&self) -> Result<Option<Box<dyn Adapter>>> {
let resolved = self
.0
.programs
.iter()
.map(|program| {
let path = on_path(&program.name).with_context(|| {
format!("package {} requires {} on PATH", self.0.name, program.name)
})?;
identify(&path, program)?;
Ok((program.name.clone(), path))
})
.collect::<Result<Vec<_>>>()?;
Ok(Some(Box::new(PackageManifestAdapter {
manifest: self.0,
resolved,
})))
}
fn install(&self) -> Result<()> {
let m = self.0;
refuse_foreign_program(m)?;
let (_root, unpack) = stage(m)?;
let bin = managed_bin_dir().context("HOME is required to install a stack member")?;
fs::create_dir_all(&bin).with_context(|| format!("create {}", bin.display()))?;
for program in &m.programs {
let source = find_program(&unpack, &program.name).with_context(|| {
format!("{} release does not contain `{}`", m.name, program.name)
})?;
install_atomically(&source, &bin.join(&program.name))?;
}
Ok(())
}
fn version(&self) -> Result<String> {
let m = self.0;
let path = resolve(&m.programs[0])?;
let out = checked(
path.as_os_str(),
m.version_argv.iter().map(String::as_str),
None,
)?;
let version = String::from_utf8_lossy(&out.stdout).trim().to_owned();
if version.is_empty() {
bail!(
"{} {} returned no version",
m.name,
m.version_argv.join(" ")
);
}
Ok(version)
}
fn is_current(&self, installed: &str) -> bool {
reports_version(installed, &self.0.version)
}
fn verify(&self) -> Result<()> {
let m = self.0;
let resolved = m
.programs
.iter()
.map(|p| Ok((p.name.as_str(), resolve(p)?)))
.collect::<Result<Vec<_>>>()?;
verify_resolved(m, &resolved)
}
}
struct PackageManifestAdapter {
manifest: &'static Manifest,
resolved: Vec<(String, PathBuf)>,
}
impl Adapter for PackageManifestAdapter {
fn name(&self) -> ToolName {
ToolName::Member(self.manifest.name.as_str())
}
fn desired_version(&self) -> String {
format!("{} {}", self.manifest.name, self.manifest.version)
}
fn install(&self) -> Result<()> {
bail!("package installation belongs to the package manager")
}
fn version(&self) -> Result<String> {
let out = checked(
self.resolved[0].1.as_os_str(),
self.manifest.version_argv.iter().map(String::as_str),
None,
)?;
let version = String::from_utf8_lossy(&out.stdout).trim().to_owned();
if version.is_empty() {
bail!("{} returned no package version", self.manifest.name);
}
Ok(version)
}
fn is_current(&self, installed: &str) -> bool {
reports_version(installed, &self.manifest.version)
}
fn verify(&self) -> Result<()> {
let resolved = self
.resolved
.iter()
.map(|(name, path)| (name.as_str(), path.clone()))
.collect::<Vec<_>>();
verify_resolved(self.manifest, &resolved)
}
}
pub(crate) fn verify_at(m: &Manifest, path: &Path) -> Result<()> {
let program = single_program(m)?;
identify_member(m, path)?;
verify_resolved(m, &[(program.name.as_str(), path.to_path_buf())])
}
pub(crate) fn identify_member(m: &Manifest, path: &Path) -> Result<()> {
identify(path, single_program(m)?)
.with_context(|| format!("refusing to update it as {}", m.name))
}
pub(crate) fn single_program(m: &Manifest) -> Result<&Program> {
match m.programs.as_slice() {
[only] => Ok(only),
_ => bail!(
"{} has {} programs; update-release tracks single-program members only",
m.name,
m.programs.len()
),
}
}
fn verify_resolved(m: &Manifest, resolved: &[(&str, PathBuf)]) -> Result<()> {
{
let root =
tempfile::tempdir().with_context(|| format!("create {} verification root", m.name))?;
let marker = format!(
"caboodle-{}-{}",
m.name,
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_nanos())
);
for (i, step) in m.verify.iter().enumerate() {
let mut argv: Vec<String> = step
.argv
.iter()
.map(|a| a.replace("{marker}", &marker))
.collect();
if let Some((_, path)) = resolved.iter().find(|(name, _)| *name == argv[0]) {
argv[0] = path.to_string_lossy().into_owned();
}
let stdin = step.stdin.as_ref().map(|s| s.replace("{marker}", &marker));
let out =
hermetic_via(root.path(), None, &argv, stdin.as_deref()).with_context(|| {
format!("{} verify step {} ({})", m.name, i + 1, argv.join(" "))
})?;
let stdout = String::from_utf8_lossy(&out.stdout);
if let Some(absent) = &step.absent {
let needle = absent.replace("{marker}", &marker);
if stdout.contains(&needle) {
bail!("{} verify step {}: control failed, {needle:?} present before it was created", m.name, i + 1);
}
}
if let Some(present) = &step.present {
let needle = present.replace("{marker}", &marker);
if !stdout.contains(&needle) {
bail!(
"{} verify step {}: {needle:?} not present after it was created",
m.name,
i + 1
);
}
}
}
Ok(())
}
}
fn reports_version(line: &str, version: &str) -> bool {
line.split_whitespace()
.any(|word| word.trim_start_matches('v') == version)
}
pub(crate) fn prove_release(
manifest: &Manifest,
archive: &Path,
wrapper: Option<&Path>,
) -> Result<()> {
let scratch = tempfile::tempdir().context("create release proof directory")?;
let unpack = materialize(manifest, archive, &scratch.path().join("unpack"))?;
for program in &manifest.programs {
let path = find_program(&unpack, &program.name).with_context(|| {
format!(
"{} release does not contain `{}`",
manifest.name, program.name
)
})?;
identify_via(&path, program, wrapper).with_context(|| {
format!(
"{} {} does not answer its manifest identity; fix identity_contains in review \
(it must still match the oldest installed release) before bumping",
manifest.name, manifest.version
)
})?;
}
let first =
find_program(&unpack, &manifest.programs[0].name).context("release program vanished")?;
let mut argv = vec![first.to_string_lossy().into_owned()];
argv.extend(manifest.version_argv.iter().cloned());
let out =
hermetic_via(scratch.path(), wrapper, &argv, None).context("read the release's version")?;
let reported = String::from_utf8_lossy(&out.stdout).trim().to_owned();
if !reports_version(&reported, &manifest.version) {
bail!(
"{} release tagged {} reports version {reported:?}; refusing to pin it",
manifest.name,
manifest.version
);
}
Ok(())
}
fn venv_root(manifest: &Manifest) -> Result<PathBuf> {
let home = env::var_os("HOME").context("HOME is required for a python-wheel member")?;
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_nanos());
Ok(PathBuf::from(home)
.join(".local/share/caboodle/members")
.join(&manifest.name)
.join(format!("{}-{nanos}", manifest.version)))
}
fn select_python(archive: &Path) -> Result<String> {
const CHECK: &str = r#"
import re, sys, zipfile
with zipfile.ZipFile(sys.argv[1]) as z:
meta = next(n for n in z.namelist() if n.endswith(".dist-info/METADATA"))
req = next((l.split(":", 1)[1].strip() for l in z.read(meta).decode().splitlines()
if l.lower().startswith("requires-python:")), "")
have = sys.version_info[:3]
def ver(s):
return tuple(int(x) for x in s.split(".")) + (0,) * (3 - len(s.split(".")))
ops = {">=": lambda a, b: a >= b, ">": lambda a, b: a > b, "<=": lambda a, b: a <= b,
"<": lambda a, b: a < b, "==": lambda a, b: a == b, "!=": lambda a, b: a != b}
for clause in filter(None, (c.strip() for c in req.split(","))):
m = re.fullmatch(r"(>=|<=|==|!=|>|<)\s*(\d+(?:\.\d+){0,2})", clause)
if not m:
print("UNEVALUABLE " + req); sys.exit(3)
if not ops[m.group(1)](have, ver(m.group(2))):
print("UNSATISFIED " + req); sys.exit(2)
print("OK " + req)
"#;
let mut candidates = vec!["python3".to_owned()];
candidates.extend((9..=14).rev().map(|minor| format!("python3.{minor}")));
let mut requirement = String::new();
for python in &candidates {
let Ok(out) = Command::new(python)
.args(["-c", CHECK])
.arg(archive)
.output()
else {
continue;
};
let text = String::from_utf8_lossy(&out.stdout).trim().to_owned();
match out.status.code() {
Some(0) => return Ok(python.clone()),
Some(3) => bail!("cannot evaluate the wheel's Requires-Python ({text}); refusing"),
_ => requirement = text,
}
}
bail!(
"no python on PATH satisfies the wheel's Requires-Python ({}); install one, then retry",
requirement
.strip_prefix("UNSATISFIED ")
.unwrap_or("python3 not found")
)
}
fn materialize(manifest: &Manifest, archive: &Path, dest: &Path) -> Result<PathBuf> {
match manifest.kind {
Kind::RustRelease => {
fs::create_dir_all(dest)?;
checked(
"tar",
[
OsStr::new("-xzf"),
archive.as_os_str(),
OsStr::new("-C"),
dest.as_os_str(),
],
None,
)?;
Ok(dest.to_path_buf())
}
Kind::PythonWheel => {
let python = select_python(archive)?;
if dest.exists() {
bail!(
"{} already exists; a venv is never rebuilt in place",
dest.display()
);
}
if let Some(parent) = dest.parent() {
fs::create_dir_all(parent)?;
}
let built = (|| -> Result<()> {
checked(
&python,
[OsStr::new("-m"), OsStr::new("venv"), dest.as_os_str()],
None,
)
.with_context(|| format!("create the {} venv with {python}", manifest.name))?;
checked(
dest.join("bin/python").as_os_str(),
[
OsStr::new("-m"),
OsStr::new("pip"),
OsStr::new("install"),
OsStr::new("--quiet"),
OsStr::new("--disable-pip-version-check"),
archive.as_os_str(),
],
None,
)
.with_context(|| format!("pip install {}", manifest.name))?;
Ok(())
})();
if let Err(error) = built {
let _ = fs::remove_dir_all(dest);
return Err(error);
}
Ok(dest.join("bin"))
}
}
}
pub(crate) fn find_program(dir: &Path, name: &str) -> Option<std::path::PathBuf> {
let direct = dir.join(name);
if direct.is_file() {
return Some(direct);
}
fs::read_dir(dir)
.ok()?
.filter_map(Result::ok)
.find_map(|e| {
let p = e.path();
if p.is_dir() {
find_program(&p, name)
} else {
None
}
})
}
fn hermetic_via(
root: &Path,
wrapper: Option<&Path>,
argv: &[String],
stdin: Option<&str>,
) -> Result<std::process::Output> {
use std::io::Write;
use std::process::Stdio;
let (program, args) = argv.split_first().context("empty verify step")?;
let mut command = hermetic_command(root, wrapper, program)?;
command
.args(args)
.stdin(if stdin.is_some() {
Stdio::piped()
} else {
Stdio::null()
})
.stdout(Stdio::piped())
.stderr(Stdio::piped());
let mut child = command.spawn().with_context(|| format!("run {program}"))?;
if let Some(text) = stdin {
child
.stdin
.take()
.context("stdin pipe")?
.write_all(text.as_bytes())
.with_context(|| format!("write stdin to {program}"))?;
}
let out = child
.wait_with_output()
.with_context(|| format!("run {program}"))?;
if !out.status.success() {
bail!(
"exited {}: {}",
out.status,
String::from_utf8_lossy(&out.stderr).trim()
);
}
Ok(out)
}
fn hermetic_command(
root: &Path,
wrapper: Option<&Path>,
program: impl AsRef<OsStr>,
) -> Result<Command> {
let mut command = match wrapper {
Some(wrapper) => {
let mut c = Command::new(wrapper);
c.arg(program);
c
}
None => Command::new(program),
};
command.current_dir(root).env_clear();
for var in ["PATH", "LANG", "LC_ALL", "TERM"] {
if let Some(value) = env::var_os(var) {
command.env(var, value);
}
}
for (var, dir) in [
("HOME", "home"),
("XDG_CONFIG_HOME", "config"),
("XDG_DATA_HOME", "data"),
("XDG_STATE_HOME", "state"),
("XDG_CACHE_HOME", "cache"),
] {
let path = root.join(dir);
fs::create_dir_all(&path)?;
command.env(var, path);
}
Ok(command)
}
#[cfg(test)]
mod python_tests {
use super::*;
fn wheel(dir: &Path, requires: &str) -> PathBuf {
let path = dir.join("demo-1.0.0-py3-none-any.whl");
let line = if requires.is_empty() {
String::new()
} else {
format!("Requires-Python: {requires}\\n")
};
let meta = format!("Metadata-Version: 2.1\\nName: demo\\nVersion: 1.0.0\\n{line}");
let script = format!(
"import sys, zipfile\nz = zipfile.ZipFile(sys.argv[1], 'w')\nz.writestr('demo-1.0.0.dist-info/METADATA', '{meta}')\nz.close()\n"
);
let status = Command::new("python3")
.args(["-c", &script])
.arg(&path)
.status()
.unwrap();
assert!(status.success());
path
}
#[test]
fn a_satisfied_or_absent_requirement_selects_an_interpreter() {
let dir = tempfile::tempdir().unwrap();
assert!(select_python(&wheel(dir.path(), ">=3.0")).is_ok());
let dir = tempfile::tempdir().unwrap();
assert!(select_python(&wheel(dir.path(), "")).is_ok());
}
#[test]
fn an_unsatisfiable_requirement_is_refused_and_named() {
let dir = tempfile::tempdir().unwrap();
let err = select_python(&wheel(dir.path(), ">=3.0,<3.1"))
.unwrap_err()
.to_string();
assert!(
err.contains("satisfies the wheel's Requires-Python (>=3.0,<3.1)"),
"{err}"
);
}
#[test]
fn an_unevaluable_requirement_is_refused_not_guessed() {
let dir = tempfile::tempdir().unwrap();
let err = select_python(&wheel(dir.path(), "~=3.11"))
.unwrap_err()
.to_string();
assert!(err.contains("cannot evaluate"), "{err}");
}
}
#[cfg(test)]
mod release_verification_tests {
use super::*;
#[test]
fn staged_member_requires_identity_version_and_functional_roundtrip() {
let mut manifest = Manifest::parse(include_str!(
"../../tests/fixtures/members/fixture-demo.toml"
))
.unwrap();
let root = tempfile::tempdir().unwrap();
let archive = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("tests/fixtures/releases/fixture-demo-v0.1.0-x86_64-unknown-linux-gnu.tar.gz");
let unpack = materialize(&manifest, &archive, &root.path().join("unpack")).unwrap();
verify_staged_member(&manifest, &unpack).unwrap();
manifest.programs[0].identity_contains = "foreign identity".to_owned();
assert!(verify_staged_member(&manifest, &unpack).is_err());
manifest.programs[0].identity_contains =
"fixture-demo is the caboodle fixture stack member".to_owned();
manifest.version = "0.1.1".to_owned();
assert!(verify_staged_member(&manifest, &unpack).is_err());
manifest.version = "0.1.0".to_owned();
manifest.verify[2].present = Some("missing-positive-control".to_owned());
assert!(verify_staged_member(&manifest, &unpack).is_err());
}
}