use std::env;
use std::path::PathBuf;
use std::process;
const PREBUILT: &str = include_str!("prebuilt.toml");
const ARCHIVE: &str = "libneedle.a";
fn main() {
println!("cargo::rerun-if-changed=build.rs");
println!("cargo::rerun-if-changed=prebuilt.toml");
println!("cargo::rerun-if-env-changed=CACTUS_NEEDLE_LIB_DIR");
println!("cargo::rerun-if-env-changed=CACTUS_CXXSTDLIB");
println!("cargo::rerun-if-env-changed=DOCS_RS");
println!("cargo::rerun-if-env-changed=HF_ENDPOINT");
println!("cargo::rerun-if-env-changed=HF_HUB_OFFLINE");
println!(
"cargo::rustc-env=NEEDLE_ENGINE_COMMIT={}",
prebuilt("commit")
);
if !feature("NEEDLE") {
return;
}
if env::var_os("DOCS_RS").is_some() {
return;
}
let lib_dir = match env::var_os("CACTUS_NEEDLE_LIB_DIR") {
Some(dir) => lib_dir_from_env(PathBuf::from(dir)),
None => downloaded_lib_dir(),
};
println!("cargo::rustc-link-search=native={}", lib_dir.display());
println!("cargo::rustc-link-lib=static=needle");
if cfg("CARGO_CFG_TARGET_OS") == "linux" {
libcxx_compat();
}
for lib in cxx_runtime() {
println!("cargo::rustc-link-lib={lib}");
}
}
fn fail(message: &str) -> ! {
for line in message.lines() {
println!("cargo::error={line}");
}
process::exit(1);
}
fn feature(name: &str) -> bool {
env::var_os(format!("CARGO_FEATURE_{name}")).is_some()
}
fn cfg(name: &str) -> String {
env::var(name).unwrap_or_default()
}
fn prebuilt(key: &str) -> &'static str {
let found = PREBUILT.lines().find_map(|line| {
let line = line.trim();
if line.starts_with('#') {
return None;
}
let (name, value) = line.split_once('=')?;
(name.trim() == key).then(|| value.trim().trim_matches('"'))
});
found.unwrap_or_else(|| {
fail(&format!(
"cactus-sys: prebuilt.toml has no entry for `{key}`.\n\
The pinned release table is incomplete or was edited by hand.\n\
Help: restore cactus-sys/prebuilt.toml from the repository."
))
})
}
fn lib_dir_from_env(dir: PathBuf) -> PathBuf {
if dir.join(ARCHIVE).is_file() {
return dir;
}
fail(&format!(
"cactus-sys: CACTUS_NEEDLE_LIB_DIR is set to `{}`, but `{ARCHIVE}` is not in it.\n\
The variable must name the directory that contains the archive, not the archive itself.\n\
Help: point it at a directory holding {ARCHIVE}, or unset it and build with the\n\
`download-binaries` feature to fetch the pinned archive automatically.",
dir.display()
))
}
#[cfg(not(feature = "download-binaries"))]
fn downloaded_lib_dir() -> PathBuf {
fail(
"cactus-sys: no engine archive available.\n\
The `needle` feature is on, but `download-binaries` is off and CACTUS_NEEDLE_LIB_DIR\n\
is unset, so there is nothing to link against. Cactus Compute ships Needle as a\n\
closed-source static library; this crate cannot build one from source.\n\
Help: either set CACTUS_NEEDLE_LIB_DIR to a directory containing libneedle.a for your\n\
target, or enable the `download-binaries` feature to fetch the pinned, SHA-256 verified\n\
archive from Hugging Face.",
)
}
#[cfg(feature = "download-binaries")]
fn downloaded_lib_dir() -> PathBuf {
let platform = platform();
let expected = prebuilt(&platform);
let out_dir = match env::var_os("OUT_DIR") {
Some(dir) => PathBuf::from(dir).join(&platform),
None => fail(
"cactus-sys: OUT_DIR is unset.\n\
Build scripts are always given one, so this build was not started by Cargo.\n\
Help: build the crate with `cargo build`.",
),
};
let archive = out_dir.join(ARCHIVE);
if download::digest(&archive).as_deref() == Some(expected) {
return out_dir;
}
if download::offline(env::var("HF_HUB_OFFLINE").ok().as_deref()) {
fail(&format!(
"cactus-sys: offline mode is on (HF_HUB_OFFLINE is set), and the Needle engine\n\
archive for `{platform}` has not been downloaded into this target directory yet.\n\
Help: set CACTUS_NEEDLE_LIB_DIR to a directory holding libneedle.a for your target,\n\
or unset HF_HUB_OFFLINE (or set it to 0) to allow the download."
));
}
let endpoint = download::endpoint(env::var("HF_ENDPOINT").ok().as_deref());
let url = download::url_with(
&endpoint,
prebuilt("repo"),
prebuilt("commit"),
&format!("{platform}/{ARCHIVE}"),
);
download::fetch(&url, &out_dir, &archive, expected);
out_dir
}
#[cfg(feature = "download-binaries")]
fn platform() -> String {
let os = cfg("CARGO_CFG_TARGET_OS");
let arch = cfg("CARGO_CFG_TARGET_ARCH");
let abi = cfg("CARGO_CFG_TARGET_ABI");
let target_env = cfg("CARGO_CFG_TARGET_ENV");
let endian = cfg("CARGO_CFG_TARGET_ENDIAN");
let platform = match (os.as_str(), arch.as_str()) {
("macos", "aarch64") => "macos-arm64",
("macos", "x86_64") => fail(
"cactus-sys: Intel macOS is not supported.\n\
Cactus Compute publishes no libneedle.a for x86_64-apple-darwin; the only macOS\n\
archive upstream ships is arm64.\n\
Help: build for aarch64-apple-darwin, or set CACTUS_NEEDLE_LIB_DIR to a directory\n\
holding an x86_64 libneedle.a you built or obtained yourself.",
),
("linux", _) if target_env == "gnu" || target_env == "musl" => {
match (arch.as_str(), endian.as_str()) {
("x86_64", _) => "linux-x86_64",
("aarch64", _) => "linux-arm64",
("arm", _) => "linux-armv7",
("riscv64", _) => "linux-riscv64",
("mips", "little") => "linux-mipsel",
_ => fail(&unsupported(&os, &arch)),
}
}
("android", "aarch64") => "android-arm64",
("android", "arm") => "android-armv7",
("android", "riscv64") => "android-riscv64",
("ios", "aarch64") if abi == "sim" => "ios-sim-arm64",
("ios", "aarch64") if abi.is_empty() => "ios-arm64",
("tvos", "aarch64") => "tvos-arm64",
("watchos", "aarch64") => "watchos-arm64",
("windows", _) if target_env == "msvc" => fail(
"cactus-sys: MSVC Windows targets are not supported.\n\
The upstream Windows archives are llvm-mingw builds: they use the Itanium C++ ABI\n\
and LLVM libc++, which the MSVC toolchain cannot link.\n\
Help: build for x86_64-pc-windows-gnullvm or aarch64-pc-windows-gnullvm.",
),
("windows", _) if abi != "llvm" => fail(
"cactus-sys: GNU (mingw-w64) Windows targets are not supported.\n\
The upstream Windows archives are llvm-mingw builds against LLVM libc++, not the\n\
libstdc++ that the *-pc-windows-gnu targets link.\n\
Help: build for x86_64-pc-windows-gnullvm or aarch64-pc-windows-gnullvm.",
),
("windows", "x86_64") => "windows-x86_64",
("windows", "aarch64") => "windows-arm64",
_ => fail(&unsupported(&os, &arch)),
};
platform.to_owned()
}
#[cfg(feature = "download-binaries")]
fn unsupported(os: &str, arch: &str) -> String {
format!(
"cactus-sys: no prebuilt Needle engine for this target (os `{os}`, arch `{arch}`).\n\
Cactus Compute publishes archives for macOS arm64, Linux (x86_64, arm64, armv7,\n\
riscv64, mipsel), Android (arm64, armv7, riscv64), iOS, iOS Simulator, tvOS, watchOS\n\
and llvm-mingw Windows (x86_64, arm64).\n\
Help: build for one of those targets, or set CACTUS_NEEDLE_LIB_DIR to a directory\n\
holding a libneedle.a for this one."
)
}
fn libcxx_compat() {
println!("cargo::rerun-if-changed=shim/libcxx_compat.c");
cc::Build::new()
.file("shim/libcxx_compat.c")
.warnings(true)
.compile("cactus_libcxx_compat");
}
fn cxx_runtime() -> Vec<String> {
if let Ok(list) = env::var("CACTUS_CXXSTDLIB") {
return list
.split(',')
.map(str::trim)
.filter(|lib| !lib.is_empty())
.map(|lib| match lib.split_once('=') {
Some(_) => lib.to_owned(),
None => format!("dylib={lib}"),
})
.collect();
}
let libs: &[&str] = match cfg("CARGO_CFG_TARGET_OS").as_str() {
"macos" | "ios" | "tvos" | "watchos" | "visionos" => &["c++"],
"linux" => {
require_libcxx();
&["c++", "c++abi"]
}
"android" => &["c++_shared"],
"windows" => &["c++"],
_ => &[],
};
libs.iter().map(|lib| format!("dylib={lib}")).collect()
}
fn require_libcxx() {
if env::var("HOST").ok() != env::var("TARGET").ok() {
return;
}
let compiler = env::var("CC").unwrap_or_else(|_| "cc".to_owned());
let found = |library: &str| {
let output = process::Command::new(&compiler)
.arg(format!("-print-file-name={library}"))
.output()
.ok()?;
Some(String::from_utf8_lossy(&output.stdout).trim().contains('/'))
};
let (Some(shared), Some(archive)) = (found("libc++.so"), found("libc++.a")) else {
return;
};
if shared || archive {
return;
}
fail(
"cactus-sys: LLVM libc++ was not found.\n\
The Needle engine archive is built against libc++, and this system appears to have only\n\
GNU libstdc++, which cannot stand in for it.\n\
Help: install libc++ (Debian and Ubuntu: `sudo apt install libc++-dev libc++abi-dev`;\n\
Fedora: `sudo dnf install libcxx-devel libcxxabi-devel`), or set CACTUS_CXXSTDLIB to the\n\
libraries to link instead.",
)
}
#[cfg(feature = "download-binaries")]
mod download {
use std::env;
use std::fmt::Write as _;
use std::fs;
use std::io::{self, Read as _};
use std::path::{Path, PathBuf};
use std::thread;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use sha2::{Digest as _, Sha256};
const MAX_BODY: u64 = 64 * 1024 * 1024;
const CONNECT_TIMEOUT: Duration = Duration::from_secs(30);
const RESPONSE_TIMEOUT: Duration = Duration::from_secs(60);
const BODY_TIMEOUT: Duration = Duration::from_secs(900);
const RETRIES: u32 = 3;
const MAX_RETRY_AFTER: Duration = Duration::from_secs(60);
const STALE_AFTER: Duration = Duration::from_secs(60 * 60);
const HF_DEFAULT_ENDPOINT: &str = "https://huggingface.co";
pub fn url_with(endpoint: &str, repo: &str, revision: &str, file: &str) -> String {
format!(
"{}/{repo}/resolve/{revision}/{file}",
endpoint.trim_end_matches('/')
)
}
pub fn endpoint(value: Option<&str>) -> String {
match value.map(|value| value.trim().trim_end_matches('/')) {
Some(value) if !value.is_empty() => value.to_owned(),
_ => HF_DEFAULT_ENDPOINT.to_owned(),
}
}
pub fn offline(value: Option<&str>) -> bool {
value.is_some_and(|value| {
let value = value.trim();
!value.is_empty() && value != "0"
})
}
enum Failure {
Transient {
detail: String,
retry_after: Option<Duration>,
},
Fatal(String),
}
pub fn digest(path: &Path) -> Option<String> {
let mut file = fs::File::open(path).ok()?;
let mut hasher = Sha256::new();
let mut buffer = [0u8; 64 * 1024];
loop {
let read = file.read(&mut buffer).ok()?;
if read == 0 {
break;
}
hasher.update(&buffer[..read]);
}
let mut hex = String::with_capacity(64);
for byte in hasher.finalize() {
write!(hex, "{byte:02x}").expect("writing to a String cannot fail");
}
Some(hex)
}
pub fn fetch(url: &str, out_dir: &Path, archive: &Path, expected: &str) {
if let Err(error) = fs::create_dir_all(out_dir) {
super::fail(&format!(
"cactus-sys: could not create `{}`: {error}\n\
Help: check the permissions on the Cargo target directory.",
out_dir.display()
));
}
remove_stale_partials(out_dir, super::ARCHIVE, SystemTime::now());
let agent = agent(body_timeout(
env::var("CACTUS_DOWNLOAD_TIMEOUT").ok().as_deref(),
));
let attempts = attempts(env::var("CACTUS_DOWNLOAD_RETRIES").ok().as_deref());
let mut mismatches = 0;
let result = retrying(attempts, thread::sleep, |_| {
let partial = partial_path(archive);
if let Err(failure) = get(&agent, url, &partial) {
let _ = fs::remove_file(&partial);
return Err(failure);
}
let actual = digest(&partial).unwrap_or_default();
if actual == expected {
return Ok(partial);
}
let _ = fs::remove_file(&partial);
mismatches += 1;
if mismatches == 1 {
return Err(Failure::Transient {
detail: format!("checksum mismatch: expected {expected}, got {actual}"),
retry_after: None,
});
}
Err(Failure::Fatal(format!(
"cactus-sys: checksum mismatch for the downloaded engine archive, twice.\n\
url: {url}\n\
expected: {expected}\n\
actual: {actual}\n\
The download was discarded. Either a proxy served something else, or\n\
prebuilt.toml no longer matches the pinned commit.\n\
Help: retry the build; if it keeps failing, verify cactus-sys/prebuilt.toml\n\
against the upstream repository before trusting the archive."
)))
});
let partial = match result {
Ok(partial) => partial,
Err(Failure::Fatal(message)) => super::fail(&message),
Err(Failure::Transient { detail, .. }) => super::fail(&format!(
"cactus-sys: could not download the Needle engine archive.\n\
url: {url}\n\
error: gave up after {attempts} attempts; the last failed with: {detail}\n\
Help: check network access to the hub (HF_ENDPOINT, default huggingface.co),\n\
raise CACTUS_DOWNLOAD_TIMEOUT or CACTUS_DOWNLOAD_RETRIES, or build offline by\n\
setting CACTUS_NEEDLE_LIB_DIR to a directory holding libneedle.a for your target."
)),
};
if let Err(error) = fs::rename(&partial, archive) {
let _ = fs::remove_file(&partial);
super::fail(&format!(
"cactus-sys: could not install `{}`: {error}\n\
Help: remove the file and rebuild.",
archive.display()
));
}
}
fn partial_path(archive: &Path) -> PathBuf {
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |elapsed| elapsed.as_nanos());
let name = archive
.file_name()
.map_or_else(|| "archive".into(), |name| name.to_string_lossy());
archive.with_file_name(format!("{name}.{}.{nanos}.partial", std::process::id()))
}
fn retrying<T>(
attempts: u32,
mut sleep: impl FnMut(Duration),
mut attempt: impl FnMut(u32) -> Result<T, Failure>,
) -> Result<T, Failure> {
let mut number = 0;
loop {
match attempt(number) {
Err(Failure::Transient { retry_after, .. }) if number + 1 < attempts => {
sleep(retry_after.unwrap_or_else(|| backoff(number, clock_nanos())));
number += 1;
}
done => return done,
}
}
}
fn backoff(attempt: u32, nanos: u32) -> Duration {
let base = 1000_u64 << attempt.min(6);
let millis = base * 3 / 4 + u64::from(nanos) % (base / 2 + 1);
Duration::from_millis(millis)
}
fn clock_nanos() -> u32 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |elapsed| elapsed.subsec_nanos())
}
fn should_retry(status: Option<u16>) -> bool {
match status {
None => true,
Some(code) => code == 408 || code == 429 || (500..600).contains(&code),
}
}
fn retry_after(value: Option<&str>) -> Option<Duration> {
let seconds = value?.trim().parse::<u64>().ok()?;
Some(Duration::from_secs(seconds).min(MAX_RETRY_AFTER))
}
fn attempts(value: Option<&str>) -> u32 {
let retries = value
.and_then(|value| value.trim().parse::<u32>().ok())
.unwrap_or(RETRIES);
retries.saturating_add(1)
}
fn body_timeout(value: Option<&str>) -> Duration {
value
.and_then(|value| value.trim().parse::<u64>().ok())
.filter(|&seconds| seconds > 0)
.map_or(BODY_TIMEOUT, Duration::from_secs)
}
fn is_stale_partial(name: &str, file: &str, modified: SystemTime, now: SystemTime) -> bool {
let ours = name
.strip_prefix(file)
.and_then(|rest| rest.strip_prefix('.'))
.is_some_and(|rest| rest == "partial" || rest.ends_with(".partial"));
ours && now
.duration_since(modified)
.is_ok_and(|age| age > STALE_AFTER)
}
fn remove_stale_partials(dir: &Path, file: &str, now: SystemTime) {
let Ok(entries) = fs::read_dir(dir) else {
return;
};
for entry in entries.flatten() {
let name = entry.file_name();
let Some(name) = name.to_str() else {
continue;
};
let Ok(modified) = entry.metadata().and_then(|metadata| metadata.modified()) else {
continue;
};
if is_stale_partial(name, file, modified, now) {
let _ = fs::remove_file(entry.path());
}
}
}
fn agent(body_timeout: Duration) -> ureq::Agent {
let config = ureq::Agent::config_builder()
.user_agent(concat!("cactus-sys/", env!("CARGO_PKG_VERSION")))
.http_status_as_error(false)
.timeout_connect(Some(CONNECT_TIMEOUT))
.timeout_recv_response(Some(RESPONSE_TIMEOUT))
.timeout_recv_body(Some(body_timeout))
.build();
ureq::Agent::new_with_config(config)
}
fn get(agent: &ureq::Agent, url: &str, dest: &Path) -> Result<(), Failure> {
let mut response = agent
.get(url)
.call()
.map_err(|error| transport(error, url))?;
let status = response.status().as_u16();
if status == 404 {
return Err(Failure::Fatal(format!(
"cactus-sys: the Needle engine archive is not where the pin says (HTTP 404).\n\
url: {url}\n\
The pinned revision or the file is missing from the repository, so the pin in\n\
prebuilt.toml is wrong.\n\
Help: restore cactus-sys/prebuilt.toml from the repository, or set\n\
CACTUS_NEEDLE_LIB_DIR to a directory holding libneedle.a for your target."
)));
}
if !(200..300).contains(&status) {
if !should_retry(Some(status)) {
return Err(Failure::Fatal(format!(
"cactus-sys: could not download the Needle engine archive.\n\
url: {url}\n\
error: HTTP {status}\n\
Help: check access to the hub (HF_ENDPOINT, default huggingface.co), or\n\
build offline by setting CACTUS_NEEDLE_LIB_DIR to a directory holding\n\
libneedle.a for your target."
)));
}
let wait = response
.headers()
.get("retry-after")
.and_then(|value| value.to_str().ok());
return Err(Failure::Transient {
detail: format!("HTTP {status}"),
retry_after: retry_after(wait),
});
}
let mut file = match fs::File::create(dest) {
Ok(file) => file,
Err(error) => {
return Err(Failure::Fatal(format!(
"cactus-sys: could not write `{}`: {error}\n\
Help: check the free space and permissions on the Cargo target directory.",
dest.display()
)));
}
};
let mut body = response.body_mut().with_config().limit(MAX_BODY).reader();
match io::copy(&mut body, &mut file) {
Ok(_) => Ok(()),
Err(error)
if error
.get_ref()
.is_some_and(|inner| inner.is::<ureq::Error>()) =>
{
Err(transport(ureq::Error::from(error), url))
}
Err(error) => Err(Failure::Fatal(format!(
"cactus-sys: could not write `{}`: {error}\n\
Help: check the free space and permissions on the Cargo target directory.",
dest.display()
))),
}
}
fn transport(error: ureq::Error, url: &str) -> Failure {
let fatal = matches!(
error,
ureq::Error::BodyExceedsLimit(_)
| ureq::Error::BadUri(_)
| ureq::Error::Http(_)
| ureq::Error::InvalidProxyUrl
| ureq::Error::RequireHttpsOnly(_)
);
if fatal {
Failure::Fatal(format!(
"cactus-sys: could not download the Needle engine archive.\n\
url: {url}\n\
error: {error}\n\
Help: check network access to the hub (HF_ENDPOINT, default huggingface.co), or\n\
build offline by setting CACTUS_NEEDLE_LIB_DIR to a directory holding\n\
libneedle.a for your target."
))
} else {
Failure::Transient {
detail: error.to_string(),
retry_after: None,
}
}
}
}