use std::ffi::{OsStr, OsString};
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context, Result};
use tokio::sync::Semaphore;
use uuid::Uuid;
use crate::shared::i18n::Locale;
const WASMER_BIN: &str = if cfg!(windows) {
"wasmer.exe"
} else {
"wasmer"
};
const DEFAULT_PYTHON_PKG: &str = "python/python";
const GUEST_WORK: &str = "/w";
const JOB_SCRIPT: &str = "job.py";
pub(crate) const GUEST_SITE: &str = "/sp";
pub(crate) const SANDBOX_IMAGE: &str = "packed-sandbox.webc";
const ENV_WASMER: &str = "MINDFORK_SANDBOX_WASMER";
const ENV_PYTHON: &str = "MINDFORK_SANDBOX_PYTHON";
const SETSOCKOPT_SHIM: &str = "\
def _mf_patch_socket():
import socket
_orig = socket.socket.setsockopt
def _safe(self, *a, **k):
try:
return _orig(self, *a, **k)
except OSError:
return None
socket.socket.setsockopt = _safe
_mf_patch_socket()
";
const MATPLOTLIB_SHIM: &str = "\
def _mf_prepare_matplotlib():
import os
d = os.environ.setdefault('MPLCONFIGDIR', '/tmp/matplotlib')
try:
os.makedirs(d, exist_ok=True)
with open(os.path.join(d, 'matplotlibrc'), 'w') as f:
f.write('backend: Agg\\ntext.hinting: default\\n')
except OSError:
pass
_mf_prepare_matplotlib()
";
#[derive(Debug, Clone, PartialEq, Default)]
pub struct SandboxOutput {
pub stdout: String,
pub stderr: String,
pub exit_code: Option<i32>,
pub timed_out: bool,
pub files: Vec<OutputFile>,
pub skipped: Vec<SkippedOutput>,
pub net_refused: bool,
}
fn strip_net_prompt(stdout: &str) -> (String, bool) {
let is_prompt = |line: &str| {
let l = line.to_ascii_lowercase();
l.contains("networking access") || (l.contains("--net") && l.contains("flag"))
};
if !stdout.lines().any(is_prompt) {
return (stdout.to_string(), false);
}
let kept: Vec<&str> = stdout.lines().filter(|l| !is_prompt(l)).collect();
let mut kept = kept.join("\n");
if stdout.ends_with('\n') && !kept.is_empty() {
kept.push('\n');
}
(kept, true)
}
#[derive(Debug, Clone, PartialEq)]
pub struct OutputFile {
pub name: String,
pub bytes: Vec<u8>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct SkippedOutput {
pub name: String,
pub reason: SkipReason,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SkipReason {
Directory,
NotAFile,
TooLarge,
TooMany,
OverTotal,
Unreadable,
TimedOut,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct OutputLimits {
pub max_files: usize,
pub max_file_bytes: u64,
pub max_total_bytes: u64,
}
impl OutputLimits {
pub const DEFAULT: Self = Self {
max_files: 10,
max_file_bytes: 25 * 1024 * 1024,
max_total_bytes: 50 * 1024 * 1024,
};
}
#[derive(Debug, Clone, PartialEq)]
pub struct SandboxInput {
pub name: String,
pub source: InputSource,
}
#[derive(Debug, Clone, PartialEq)]
pub enum InputSource {
Bytes(Vec<u8>),
Path(PathBuf),
}
impl SandboxInput {
pub fn bytes(name: impl Into<String>, bytes: Vec<u8>) -> Self {
Self {
name: name.into(),
source: InputSource::Bytes(bytes),
}
}
pub fn path(name: impl Into<String>, path: impl Into<PathBuf>) -> Self {
Self {
name: name.into(),
source: InputSource::Path(path.into()),
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct SandboxJob<'a> {
pub code: &'a str,
pub inputs: &'a [SandboxInput],
pub net: bool,
pub timeout: Duration,
}
impl<'a> SandboxJob<'a> {
pub fn new(code: &'a str, net: bool, timeout: Duration) -> Self {
Self {
code,
inputs: &[],
net,
timeout,
}
}
pub fn with_inputs(mut self, inputs: &'a [SandboxInput]) -> Self {
self.inputs = inputs;
self
}
}
fn is_one_component(name: &str) -> bool {
!name.is_empty() && name != "." && name != ".." && !name.contains(['/', '\\', ':', '\0'])
}
#[derive(Debug, Clone, PartialEq)]
pub enum SandboxAvailability {
Ready,
Missing(String),
}
#[async_trait::async_trait]
pub trait SandboxRunner: Send + Sync {
fn availability(&self, loc: &Locale) -> SandboxAvailability;
async fn run(&self, job: SandboxJob<'_>, loc: &Locale) -> Result<SandboxOutput>;
}
#[derive(Debug, Clone, Copy, PartialEq)]
enum SiteSource {
Image,
Directory,
}
#[derive(Debug, PartialEq)]
struct LaunchPlan {
program: OsString,
site_mount: Option<PathBuf>,
site_on_path: bool,
}
pub struct WasmerSandbox {
dir: Option<PathBuf>,
site: SiteSource,
image: String,
gate: Arc<Semaphore>,
memory_mb: Option<u64>,
allow_private: bool,
}
impl WasmerSandbox {
pub fn new(dir: Option<PathBuf>) -> Self {
Self::with_site(dir, SiteSource::Image, SANDBOX_IMAGE)
}
pub fn for_candidate(dir: PathBuf, image: &str) -> Self {
Self::with_site(Some(dir), SiteSource::Image, image)
}
pub fn for_provisioning(dir: PathBuf) -> Self {
Self::with_site(Some(dir), SiteSource::Directory, SANDBOX_IMAGE)
}
fn with_site(dir: Option<PathBuf>, site: SiteSource, image: &str) -> Self {
Self {
dir,
site,
image: image.to_string(),
gate: Arc::new(Semaphore::new(1)),
memory_mb: None,
allow_private: false,
}
}
pub fn with_memory_limit(mut self, mb: Option<u64>) -> Self {
self.memory_mb = mb.filter(|&m| m > 0);
self
}
pub fn with_private_network(mut self, allow: bool) -> Self {
self.allow_private = allow;
self
}
fn resolve_wasmer(&self) -> Option<OsString> {
if let Some(o) = env_override(ENV_WASMER) {
return Some(o);
}
self.dir
.as_deref()
.and_then(locate_wasmer)
.map(PathBuf::into_os_string)
}
fn resolve_python(&self) -> OsString {
if let Some(o) = env_override(ENV_PYTHON) {
return o;
}
if let Some(dir) = &self.dir {
let webc = dir.join("python.webc");
if webc.is_file() {
return webc.into_os_string();
}
}
OsString::from(DEFAULT_PYTHON_PKG)
}
fn plan(&self) -> Option<LaunchPlan> {
let site_dir = self
.dir
.as_ref()
.map(|d| d.join("site-packages"))
.filter(|p| p.is_dir());
if self.site == SiteSource::Directory {
return Some(LaunchPlan {
program: self.resolve_python(),
site_on_path: site_dir.is_some(),
site_mount: site_dir,
});
}
let image = self
.dir
.as_ref()
.map(|d| d.join(&self.image))
.filter(|p| p.is_file());
match (image, site_dir) {
(Some(image), _) => Some(LaunchPlan {
program: image.into_os_string(),
site_mount: None,
site_on_path: true,
}),
(None, Some(_)) => None,
(None, None) => Some(LaunchPlan {
program: self.resolve_python(),
site_mount: None,
site_on_path: false,
}),
}
}
}
#[async_trait::async_trait]
impl SandboxRunner for WasmerSandbox {
fn availability(&self, loc: &Locale) -> SandboxAvailability {
match self.resolve_wasmer() {
None => SandboxAvailability::Missing(loc.t("sandbox.err.not_installed").to_string()),
Some(_) if self.plan().is_none() => {
SandboxAvailability::Missing(loc.t("sandbox.err.needs_repack").to_string())
}
Some(_) => SandboxAvailability::Ready,
}
}
async fn run(&self, spec: SandboxJob<'_>, loc: &Locale) -> Result<SandboxOutput> {
let _permit = self
.gate
.try_acquire()
.map_err(|_| anyhow::anyhow!("{}", loc.t("sandbox.err.busy")))?;
let wasmer = self
.resolve_wasmer()
.ok_or_else(|| anyhow::anyhow!("{}", loc.t("sandbox.err.not_found")))?;
let plan = self
.plan()
.ok_or_else(|| anyhow::anyhow!("{}", loc.t("sandbox.err.needs_repack")))?;
let job = JobDir::create()
.await
.with_context(|| loc.t("sandbox.err.job_dir").to_string())?;
let layout = prepare_job(&job, &build_wrapper(spec.code), spec.inputs, loc).await?;
let out_dir = layout.out_dir;
let mut mounts: Vec<(PathBuf, &str)> = vec![(job.path.clone(), GUEST_WORK)];
let mut envs: Vec<(&str, String)> = vec![
("PYTHONIOENCODING", "utf-8".into()),
("PYTHONUTF8", "1".into()),
];
if let Some(sp) = plan.site_mount {
mounts.push((sp, GUEST_SITE));
}
if plan.site_on_path {
envs.push(("PYTHONPATH", GUEST_SITE.into()));
}
let script_guest = format!("{GUEST_WORK}/{JOB_SCRIPT}");
let net = net_arg(spec.net, self.allow_private);
let args = build_args(&plan.program, &mounts, &envs, net.as_deref(), &script_guest);
let mut cmd = tokio::process::Command::new(&wasmer);
cmd.args(&args)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true);
if let Some(dir) = &self.dir {
cmd.env("WASMER_CACHE_DIR", dir.join("cache"));
}
let child = cmd
.spawn()
.with_context(|| loc.tf("sandbox.err.spawn", &[("path", &wasmer.to_string_lossy())]))?;
if let Some(mb) = self.memory_mb {
apply_memory_limit(&child, mb);
}
match tokio::time::timeout(spec.timeout, child.wait_with_output()).await {
Ok(Ok(out)) => {
let (files, skipped) = collected(out_dir, collect_outputs).await;
let (stdout, net_refused) = strip_net_prompt(&String::from_utf8_lossy(&out.stdout));
Ok(SandboxOutput {
net_refused,
stdout,
stderr: String::from_utf8_lossy(&out.stderr).into_owned(),
exit_code: out.status.code(),
timed_out: false,
files,
skipped,
})
}
Ok(Err(e)) => Err(e).with_context(|| loc.t("sandbox.err.wait").to_string()),
Err(_) => Ok(SandboxOutput {
timed_out: true,
skipped: named_off_loop(out_dir, SkipReason::TimedOut).await,
..SandboxOutput::default()
}),
}
}
}
pub struct LocalSandbox {
python: Option<String>,
memory_mb: Option<u64>,
named_secrets: Vec<String>,
}
impl LocalSandbox {
pub fn new(python: Option<String>) -> Self {
Self {
python,
memory_mb: None,
named_secrets: Vec::new(),
}
}
pub fn with_named_secrets(mut self, names: Vec<String>) -> Self {
self.named_secrets = names;
self
}
pub fn with_memory_limit(mut self, mb: Option<u64>) -> Self {
self.memory_mb = mb.filter(|&m| m > 0);
self
}
pub fn interpreter(&self) -> String {
self.python.clone().unwrap_or_else(|| {
if cfg!(windows) {
"python".to_string()
} else {
"python3".to_string()
}
})
}
}
#[async_trait::async_trait]
impl SandboxRunner for LocalSandbox {
fn availability(&self, loc: &Locale) -> SandboxAvailability {
let python = self.interpreter();
if python.contains(['/', '\\']) && !Path::new(&python).is_file() {
return SandboxAvailability::Missing(
loc.tf("sandbox.err.python_not_found", &[("path", &python)]),
);
}
SandboxAvailability::Ready
}
async fn run(&self, spec: SandboxJob<'_>, loc: &Locale) -> Result<SandboxOutput> {
let python = self.interpreter();
let job = JobDir::create()
.await
.with_context(|| loc.t("sandbox.err.job_dir").to_string())?;
let layout = prepare_job(&job, spec.code, spec.inputs, loc).await?;
let mut cmd = tokio::process::Command::new(&python);
cmd.arg(&layout.script)
.current_dir(&job.path)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.env("PYTHONIOENCODING", "utf-8")
.env("PYTHONUTF8", "1")
.kill_on_drop(true);
for name in crate::shared::child_env::credential_vars(&self.named_secrets) {
cmd.env_remove(name);
}
let mut child = cmd
.spawn()
.with_context(|| loc.tf("sandbox.err.spawn_python", &[("path", &python)]))?;
if let Some(mb) = self.memory_mb {
apply_memory_limit(&child, mb);
}
let (out_buf, out_task) = pipe_reader(child.stdout.take().expect("stdout is piped"));
let (err_buf, err_task) = pipe_reader(child.stderr.take().expect("stderr is piped"));
match tokio::time::timeout(spec.timeout, child.wait()).await {
Ok(Ok(status)) => {
let (files, skipped) = collected(layout.out_dir, collect_outputs).await;
Ok(SandboxOutput {
stdout: drained(&out_buf, out_task).await,
stderr: drained(&err_buf, err_task).await,
exit_code: status.code(),
timed_out: false,
files,
skipped,
net_refused: false,
})
}
Ok(Err(e)) => Err(e).with_context(|| loc.t("sandbox.err.wait_python").to_string()),
Err(_) => Ok(SandboxOutput {
timed_out: true,
skipped: named_off_loop(layout.out_dir, SkipReason::TimedOut).await,
..SandboxOutput::default()
}),
}
}
}
pub fn locate_wasmer(dir: &Path) -> Option<PathBuf> {
let direct = dir.join(WASMER_BIN);
if direct.is_file() {
return Some(direct);
}
let dist = dir.join("wasmer-dist").join("bin").join(WASMER_BIN);
dist.is_file().then_some(dist)
}
fn env_override(key: &str) -> Option<OsString> {
std::env::var_os(key).filter(|v| !v.is_empty())
}
#[cfg(windows)]
fn apply_memory_limit(child: &tokio::process::Child, mb: u64) {
use windows_sys::Win32::Foundation::{CloseHandle, HANDLE};
use windows_sys::Win32::System::JobObjects::{
AssignProcessToJobObject, CreateJobObjectW, JOB_OBJECT_LIMIT_PROCESS_MEMORY,
JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JobObjectExtendedLimitInformation,
SetInformationJobObject,
};
let Some(raw) = child.raw_handle() else {
tracing::warn!("sandbox: no process handle — memory limit not applied");
return;
};
unsafe {
let job: HANDLE = CreateJobObjectW(std::ptr::null(), std::ptr::null());
if job.is_null() {
tracing::warn!("sandbox: CreateJobObjectW failed");
return;
}
let mut info: JOBOBJECT_EXTENDED_LIMIT_INFORMATION = std::mem::zeroed();
info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_PROCESS_MEMORY;
info.ProcessMemoryLimit = (mb as usize).saturating_mul(1024 * 1024);
let ok = SetInformationJobObject(
job,
JobObjectExtendedLimitInformation,
&info as *const _ as *const core::ffi::c_void,
std::mem::size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>() as u32,
);
if ok == 0 {
tracing::warn!("sandbox: SetInformationJobObject failed");
CloseHandle(job);
return;
}
if AssignProcessToJobObject(job, raw as HANDLE) == 0 {
tracing::warn!("sandbox: AssignProcessToJobObject failed");
}
CloseHandle(job);
}
}
#[cfg(not(windows))]
fn apply_memory_limit(_child: &tokio::process::Child, _mb: u64) {
tracing::debug!("sandbox: memory limit is supported only on Windows — skipping");
}
fn net_arg(net: bool, allow_private: bool) -> Option<OsString> {
match (net, allow_private) {
(false, _) => None,
(true, true) => Some("--net".into()),
(true, false) => Some(format!("--net={}", crate::shared::net::sandbox_net_rules()).into()),
}
}
pub fn build_wrapper(code: &str) -> String {
format!("{SETSOCKOPT_SHIM}{MATPLOTLIB_SHIM}\n{code}")
}
fn build_args(
python: &OsStr,
mounts: &[(PathBuf, &str)],
envs: &[(&str, String)],
net: Option<&OsStr>,
script_guest: &str,
) -> Vec<OsString> {
let mut a: Vec<OsString> = vec!["run".into(), "--v8".into()];
if let Some(net) = net {
a.push(net.to_os_string());
}
for (host, guest) in mounts {
a.push("--volume".into());
let mut v = host.clone().into_os_string();
v.push(":");
v.push(guest);
a.push(v);
}
for (k, val) in envs {
a.push("--env".into());
a.push(format!("{k}={val}").into());
}
a.push(python.to_os_string());
a.push("--".into());
a.push(script_guest.into());
a
}
async fn prepare_job(
job: &JobDir,
code: &str,
inputs: &[SandboxInput],
loc: &Locale,
) -> Result<JobLayout> {
let script = job.path.join(JOB_SCRIPT);
tokio::fs::write(&script, code)
.await
.with_context(|| loc.t("sandbox.err.write_script").to_string())?;
let out_dir = job.path.join("out");
let in_dir = job.path.join("in");
for dir in [&out_dir, &in_dir] {
tokio::fs::create_dir(dir)
.await
.with_context(|| loc.t("sandbox.err.job_dir").to_string())?;
}
for input in inputs {
if !is_one_component(&input.name) {
anyhow::bail!(
"{}",
loc.tf("sandbox.err.input_name", &[("name", &input.name)])
);
}
let to = in_dir.join(&input.name);
match &input.source {
InputSource::Bytes(bytes) => tokio::fs::write(&to, bytes).await.map(|()| 0),
InputSource::Path(from) => tokio::fs::copy(from, &to).await,
}
.with_context(|| loc.tf("sandbox.err.stage_input", &[("name", &input.name)]))?;
}
Ok(JobLayout { script, out_dir })
}
#[derive(Debug)]
struct JobLayout {
script: PathBuf,
out_dir: PathBuf,
}
struct JobDir {
path: PathBuf,
}
impl JobDir {
async fn create() -> std::io::Result<Self> {
tokio::task::spawn_blocking(|| {
sweep_stale_jobs();
let path = std::env::temp_dir().join(format!("{JOB_PREFIX}{}", Uuid::new_v4()));
std::fs::create_dir_all(&path)?;
Ok(Self { path })
})
.await
.map_err(std::io::Error::other)?
}
}
const JOB_PREFIX: &str = "mindfork-sbx-";
const JOB_STALE_AFTER: Duration = Duration::from_secs(24 * 60 * 60);
fn sweep_stale_jobs() {
static SWEPT: std::sync::Once = std::sync::Once::new();
SWEPT.call_once(|| sweep_stale_in(&std::env::temp_dir(), JOB_STALE_AFTER));
}
fn sweep_stale_in(temp: &Path, older_than: Duration) {
let Ok(entries) = std::fs::read_dir(temp) else {
return;
};
for entry in entries.filter_map(std::result::Result::ok) {
if !entry.file_name().to_string_lossy().starts_with(JOB_PREFIX) {
continue;
}
let stale = entry
.metadata()
.and_then(|m| m.modified())
.is_ok_and(|t| t.elapsed().is_ok_and(|age| age >= older_than));
if stale {
let _ = std::fs::remove_dir_all(entry.path());
}
}
}
const MAX_PIPE_BYTES: usize = 1 << 20;
const DRAIN_GRACE: Duration = Duration::from_millis(200);
fn pipe_reader<R>(mut pipe: R) -> (Arc<std::sync::Mutex<Vec<u8>>>, tokio::task::JoinHandle<()>)
where
R: tokio::io::AsyncRead + Unpin + Send + 'static,
{
let buf = Arc::new(std::sync::Mutex::new(Vec::new()));
let sink = Arc::clone(&buf);
let task = tokio::spawn(async move {
use tokio::io::AsyncReadExt;
let mut chunk = [0u8; 8 * 1024];
while let Ok(n) = pipe.read(&mut chunk).await {
if n == 0 {
break;
}
let mut held = sink.lock().expect("pipe buffer poisoned");
let room = MAX_PIPE_BYTES.saturating_sub(held.len());
if room > 0 {
held.extend_from_slice(&chunk[..n.min(room)]);
}
}
});
(buf, task)
}
async fn drained(
buf: &Arc<std::sync::Mutex<Vec<u8>>>,
mut task: tokio::task::JoinHandle<()>,
) -> String {
let _ = tokio::time::timeout(DRAIN_GRACE, &mut task).await;
task.abort();
String::from_utf8_lossy(&buf.lock().expect("pipe buffer poisoned")).into_owned()
}
impl Drop for JobDir {
fn drop(&mut self) {
let path = std::mem::take(&mut self.path);
let remove = move || {
let _ = std::fs::remove_dir_all(&path);
};
match tokio::runtime::Handle::try_current() {
Ok(runtime) => {
runtime.spawn_blocking(remove);
}
Err(_) => remove(),
}
}
}
fn collect_outputs(out: &Path, limits: OutputLimits) -> (Vec<OutputFile>, Vec<SkippedOutput>) {
let mut files: Vec<OutputFile> = Vec::new();
let mut skipped = Vec::new();
let mut total = 0u64;
for (name, path) in sorted_entries(out) {
let reason = match std::fs::symlink_metadata(&path) {
Err(_) => SkipReason::Unreadable,
Ok(meta) if meta.is_dir() => SkipReason::Directory,
Ok(meta) if !meta.is_file() => SkipReason::NotAFile,
Ok(_) if files.len() >= limits.max_files => SkipReason::TooMany,
Ok(meta) if meta.len() > limits.max_file_bytes => SkipReason::TooLarge,
Ok(_) => match read_capped(&path, limits.max_file_bytes) {
Err(_) => SkipReason::Unreadable,
Ok(None) => SkipReason::TooLarge,
Ok(Some(bytes)) if total + bytes.len() as u64 > limits.max_total_bytes => {
SkipReason::OverTotal
}
Ok(Some(bytes)) => {
total += bytes.len() as u64;
files.push(OutputFile { name, bytes });
continue;
}
},
};
skipped.push(SkippedOutput { name, reason });
}
(files, skipped)
}
fn sorted_entries(dir: &Path) -> Vec<(String, PathBuf)> {
let Ok(entries) = std::fs::read_dir(dir) else {
return Vec::new();
};
let mut entries: Vec<(String, PathBuf)> = entries
.filter_map(Result::ok)
.map(|e| (e.file_name().to_string_lossy().into_owned(), e.path()))
.collect();
entries.sort();
entries
}
fn read_capped(path: &Path, cap: u64) -> std::io::Result<Option<Vec<u8>>> {
use std::io::Read;
let mut bytes = Vec::new();
std::fs::File::open(path)?
.take(cap.saturating_add(1))
.read_to_end(&mut bytes)?;
Ok((bytes.len() as u64 <= cap).then_some(bytes))
}
fn named_outputs(out: &Path, reason: SkipReason) -> Vec<SkippedOutput> {
sorted_entries(out)
.into_iter()
.map(|(name, _)| SkippedOutput { name, reason })
.collect()
}
type Collected = (Vec<OutputFile>, Vec<SkippedOutput>);
async fn collected(out: PathBuf, collect: fn(&Path, OutputLimits) -> Collected) -> Collected {
let from = out.clone();
match tokio::task::spawn_blocking(move || collect(&from, OutputLimits::DEFAULT)).await {
Ok(collected) => collected,
Err(err) => {
tracing::error!(
error = %err,
dir = %out.display(),
"collecting a call's outputs failed; its files are reported as unreadable"
);
(
Vec::new(),
named_off_loop(out, SkipReason::Unreadable).await,
)
}
}
}
async fn named_off_loop(out: PathBuf, reason: SkipReason) -> Vec<SkippedOutput> {
tokio::task::spawn_blocking(move || named_outputs(&out, reason))
.await
.unwrap_or_else(|err| {
tracing::error!(error = %err, "listing a call's outputs failed");
Vec::new()
})
}
#[cfg(test)]
mod collect_tests {
use super::*;
fn write(dir: &Path, name: &str, bytes: &[u8]) {
std::fs::write(dir.join(name), bytes).unwrap();
}
fn names(files: &[OutputFile]) -> Vec<&str> {
files.iter().map(|f| f.name.as_str()).collect()
}
#[test]
fn collects_regular_files_in_name_order() {
let dir = tempfile::tempdir().unwrap();
write(dir.path(), "b.txt", b"bb");
write(dir.path(), "a.png", b"aa");
let (files, skipped) = collect_outputs(dir.path(), OutputLimits::DEFAULT);
assert_eq!(names(&files), ["a.png", "b.txt"]);
assert_eq!(files[0].bytes, b"aa");
assert!(skipped.is_empty());
}
#[test]
fn a_directory_is_skipped_and_named_as_one() {
let dir = tempfile::tempdir().unwrap();
let sub = dir.path().join("charts");
std::fs::create_dir(&sub).unwrap();
write(&sub, "inner.png", b"x");
let (files, skipped) = collect_outputs(dir.path(), OutputLimits::DEFAULT);
assert!(files.is_empty());
assert_eq!(
skipped,
[SkippedOutput {
name: "charts".into(),
reason: SkipReason::Directory
}]
);
}
#[cfg(unix)]
#[test]
fn a_link_is_never_followed() {
let dir = tempfile::tempdir().unwrap();
let outside = tempfile::tempdir().unwrap();
write(outside.path(), "secret.txt", b"host");
std::os::unix::fs::symlink(outside.path().join("secret.txt"), dir.path().join("l.txt"))
.unwrap();
let (files, skipped) = collect_outputs(dir.path(), OutputLimits::DEFAULT);
assert!(files.is_empty(), "a link's target was read");
assert_eq!(skipped[0].reason, SkipReason::NotAFile);
}
#[cfg(windows)]
#[test]
fn a_link_is_never_followed() {
let dir = tempfile::tempdir().unwrap();
let outside = tempfile::tempdir().unwrap();
write(outside.path(), "secret.txt", b"host");
let link = dir.path().join("l.txt");
if std::os::windows::fs::symlink_file(outside.path().join("secret.txt"), &link).is_err() {
eprintln!("skip: creating a symlink needs a privilege here");
return;
}
let (files, skipped) = collect_outputs(dir.path(), OutputLimits::DEFAULT);
assert!(files.is_empty(), "a link's target was read");
assert_eq!(skipped[0].reason, SkipReason::NotAFile);
}
#[test]
fn the_caps_skip_what_they_drop_and_name_it() {
let dir = tempfile::tempdir().unwrap();
write(dir.path(), "1.bin", b"12345678"); write(dir.path(), "2.bin", b"123456789"); write(dir.path(), "3.bin", b"1234567"); write(dir.path(), "4.bin", b"1234"); write(dir.path(), "5.bin", b"1"); write(dir.path(), "6.bin", b"1"); let limits = OutputLimits {
max_files: 3,
max_file_bytes: 8,
max_total_bytes: 14,
};
let (files, skipped) = collect_outputs(dir.path(), limits);
assert_eq!(names(&files), ["1.bin", "4.bin", "5.bin"]);
let reasons: Vec<(&str, SkipReason)> = skipped
.iter()
.map(|s| (s.name.as_str(), s.reason))
.collect();
assert_eq!(
reasons,
[
("2.bin", SkipReason::TooLarge),
("3.bin", SkipReason::OverTotal),
("6.bin", SkipReason::TooMany),
]
);
}
#[test]
fn a_timed_out_call_names_what_it_left_and_reads_none_of_it() {
let dir = tempfile::tempdir().unwrap();
write(dir.path(), "half.png", b"\x89PNG");
assert_eq!(
named_outputs(dir.path(), SkipReason::TimedOut),
[SkippedOutput {
name: "half.png".into(),
reason: SkipReason::TimedOut
}]
);
}
#[tokio::test]
async fn a_collector_that_fails_names_what_out_held_instead_of_reporting_nothing() {
fn failing(_: &Path, _: OutputLimits) -> Collected {
panic!("a collector bug, on purpose");
}
let dir = tempfile::tempdir().unwrap();
write(dir.path(), "chart.png", b"\x89PNG");
let (files, skipped) = collected(dir.path().to_path_buf(), collect_outputs).await;
assert_eq!(files.len(), 1, "{skipped:?}");
let (files, skipped) = collected(dir.path().to_path_buf(), failing).await;
assert!(files.is_empty());
assert_eq!(
skipped,
[SkippedOutput {
name: "chart.png".into(),
reason: SkipReason::Unreadable
}],
"the file is named, not forgotten"
);
}
#[test]
fn a_job_directory_is_removed_on_drop_on_a_runtime_and_off_one() {
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(1)
.enable_time()
.build()
.unwrap();
let on_runtime = runtime.block_on(async {
let job = JobDir::create().await.expect("a job dir");
std::fs::write(job.path.join("copy.csv"), b"a,b").unwrap();
let path = job.path.clone();
drop(job);
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
while tokio::fs::try_exists(&path).await.unwrap_or(false)
&& tokio::time::Instant::now() < deadline
{
tokio::time::sleep(Duration::from_millis(10)).await;
}
path
});
assert!(!on_runtime.exists(), "{}", on_runtime.display());
let job = tokio::runtime::Runtime::new()
.unwrap()
.block_on(JobDir::create())
.expect("a job dir");
let off_runtime = job.path.clone();
drop(job);
assert!(!off_runtime.exists(), "{}", off_runtime.display());
}
#[test]
fn a_missing_directory_collects_nothing() {
let dir = tempfile::tempdir().unwrap();
let (files, skipped) = collect_outputs(&dir.path().join("absent"), OutputLimits::DEFAULT);
assert!(files.is_empty() && skipped.is_empty());
}
}
#[cfg(test)]
pub type StagedFiles = Vec<(String, Vec<u8>)>;
#[cfg(test)]
pub struct MockSandbox {
availability: SandboxAvailability,
output: SandboxOutput,
pub calls: std::sync::Mutex<Vec<(String, bool)>>,
pub staged: std::sync::Mutex<Vec<StagedFiles>>,
}
#[cfg(test)]
impl MockSandbox {
pub fn ready(output: SandboxOutput) -> Self {
Self {
availability: SandboxAvailability::Ready,
output,
calls: std::sync::Mutex::new(Vec::new()),
staged: std::sync::Mutex::new(Vec::new()),
}
}
pub fn missing(reason: &str) -> Self {
Self {
availability: SandboxAvailability::Missing(reason.into()),
output: SandboxOutput::default(),
calls: std::sync::Mutex::new(Vec::new()),
staged: std::sync::Mutex::new(Vec::new()),
}
}
}
#[cfg(test)]
#[async_trait::async_trait]
impl SandboxRunner for MockSandbox {
fn availability(&self, _loc: &Locale) -> SandboxAvailability {
self.availability.clone()
}
async fn run(&self, job: SandboxJob<'_>, _loc: &Locale) -> Result<SandboxOutput> {
self.calls
.lock()
.unwrap()
.push((job.code.to_string(), job.net));
self.staged.lock().unwrap().push(
job.inputs
.iter()
.map(|input| {
let bytes = match &input.source {
InputSource::Bytes(bytes) => bytes.clone(),
InputSource::Path(from) => std::fs::read(from).unwrap_or_default(),
};
(input.name.clone(), bytes)
})
.collect(),
);
Ok(self.output.clone())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::shared::i18n::{Lang, locale};
fn ru() -> &'static Locale {
locale(Lang::Ru)
}
#[tokio::test]
async fn a_job_directory_holds_the_script_the_copies_and_an_out_folder() {
let host = tempfile::tempdir().expect("a source dir");
let from = host.path().join("sales.xlsx");
std::fs::write(&from, b"PK\x03\x04").unwrap();
let job = JobDir::create().await.expect("a job dir");
let inputs = [
SandboxInput::bytes("memo.txt", b"the note".to_vec()),
SandboxInput::path("sales.xlsx", &from),
];
let layout = prepare_job(&job, "print(1)", &inputs, ru()).await.unwrap();
assert_eq!(std::fs::read_to_string(&layout.script).unwrap(), "print(1)");
assert!(layout.out_dir.is_dir(), "out/ must exist before the run");
let staged = job.path.join("in");
assert_eq!(
std::fs::read_to_string(staged.join("memo.txt")).unwrap(),
"the note"
);
assert_eq!(
std::fs::read(staged.join("sales.xlsx")).unwrap(),
b"PK\x03\x04"
);
assert!(from.is_file());
}
#[tokio::test]
async fn a_staged_name_that_is_not_one_component_is_refused() {
for name in ["../escape.txt", "sub/dir.txt", "..", ""] {
let job = JobDir::create().await.expect("a job dir");
let inputs = [SandboxInput::bytes(name, b"x".to_vec())];
let err = prepare_job(&job, "print(1)", &inputs, ru())
.await
.expect_err("the name must be refused");
assert!(format!("{err:#}").contains("имя"), "{name:?}: {err:#}");
}
}
#[test]
fn a_local_memory_limit_of_zero_is_no_limit() {
let limit = |mb| LocalSandbox::new(None).with_memory_limit(mb).memory_mb;
assert_eq!(limit(Some(0)), None);
assert_eq!(limit(None), None);
assert_eq!(limit(Some(512)), Some(512));
}
#[test]
fn a_local_interpreter_path_is_checked_and_a_bare_name_is_not() {
let missing = LocalSandbox::new(Some("D:\\nowhere\\python.exe".into()));
assert!(matches!(
missing.availability(ru()),
SandboxAvailability::Missing(_)
));
assert_eq!(
LocalSandbox::new(None).availability(ru()),
SandboxAvailability::Ready
);
assert_eq!(
LocalSandbox::new(Some("python3".into())).availability(ru()),
SandboxAvailability::Ready
);
let default = LocalSandbox::new(None).interpreter();
assert_eq!(default, if cfg!(windows) { "python" } else { "python3" });
}
#[test]
fn wrapper_prepends_setsockopt_shim() {
let w = build_wrapper("print(1)");
assert!(w.contains("_mf_patch_socket"));
assert!(w.contains("setsockopt"));
assert!(w.trim_end().ends_with("print(1)"));
}
#[test]
fn wrapper_prepares_matplotlib_for_wasix() {
let w = build_wrapper("print(1)");
assert!(w.contains("MPLCONFIGDIR"), "{w}");
assert!(w.contains(r"text.hinting: default\n"), "{w}");
let shim = w
.find("_mf_prepare_matplotlib()")
.expect("the shim is called");
assert!(shim < w.find("print(1)").unwrap(), "{w}");
}
#[test]
fn the_runtimes_network_prompt_leaves_the_output() {
let stdout = "first line
The current package is requesting networking access. Run the package with `--net` flag to bypass the prompt.
second line
";
let (kept, refused) = strip_net_prompt(stdout);
assert!(refused);
assert_eq!(
kept,
"first line
second line
"
);
let ordinary = "result: 42
";
let (kept, refused) = strip_net_prompt(ordinary);
assert!(!refused);
assert_eq!(kept, ordinary);
}
#[test]
fn build_args_without_net_omits_flag() {
let mounts = [(PathBuf::from("/tmp/job"), GUEST_WORK)];
let envs = [("PYTHONUTF8", "1".to_string())];
let a = build_args(
OsStr::new("python/python"),
&mounts,
&envs,
None,
"/w/job.py",
);
let s: Vec<String> = a.iter().map(|x| x.to_string_lossy().into_owned()).collect();
assert_eq!(s[0], "run");
assert_eq!(s[1], "--v8");
assert!(!s.iter().any(|x| x == "--net"));
assert!(s.iter().any(|x| x == "--volume"));
assert!(s.iter().any(|x| x.ends_with(":/w")));
assert!(s.iter().any(|x| x == "--env"));
assert!(s.iter().any(|x| x == "PYTHONUTF8=1"));
assert_eq!(s[s.len() - 3], "python/python");
assert_eq!(s[s.len() - 2], "--");
assert_eq!(s[s.len() - 1], "/w/job.py");
}
#[test]
fn build_args_with_net_adds_flag_before_mounts() {
let mounts = [(PathBuf::from("/tmp/job"), GUEST_WORK)];
let net = net_arg(true, true).unwrap();
let a = build_args(
OsStr::new("python/python"),
&mounts,
&[],
Some(&net),
"/w/job.py",
);
let s: Vec<String> = a.iter().map(|x| x.to_string_lossy().into_owned()).collect();
assert_eq!(s[2], "--net");
}
#[test]
fn net_arg_carries_the_deny_rules_unless_private_is_allowed() {
assert_eq!(net_arg(false, false), None);
assert_eq!(net_arg(false, true), None);
assert_eq!(net_arg(true, true).unwrap(), OsStr::new("--net"));
let filtered = net_arg(true, false).unwrap();
let filtered = filtered.to_string_lossy().into_owned();
assert!(filtered.starts_with("--net="), "{filtered}");
for required in [
"ipv4:allow=*:*",
"dns:allow=*:*",
"ipv4:deny=127.0.0.0/8:*",
"ipv4:deny=192.168.0.0/16:*",
"ipv4:deny=169.254.0.0/16:*",
"ipv6:deny=::/96:*",
"ipv6:deny=fe80::/10:*",
] {
assert!(
filtered.contains(required),
"{required} missing: {filtered}"
);
}
}
#[test]
fn build_args_mount_is_host_colon_guest() {
let mounts = [(PathBuf::from("/home/u/job"), GUEST_WORK)];
let a = build_args(OsStr::new("python/python"), &mounts, &[], None, "/w/job.py");
let vol = a
.iter()
.position(|x| x == OsStr::new("--volume"))
.map(|i| a[i + 1].to_string_lossy().into_owned())
.unwrap();
assert!(vol.ends_with(":/w"), "vol = {vol}");
assert!(vol.starts_with("/home/u/job"), "vol = {vol}");
}
#[test]
fn locate_wasmer_finds_direct_and_dist() {
let dir = tempfile::tempdir().unwrap();
assert!(locate_wasmer(dir.path()).is_none());
let bin_dir = dir.path().join("wasmer-dist").join("bin");
std::fs::create_dir_all(&bin_dir).unwrap();
std::fs::write(bin_dir.join(WASMER_BIN), b"stub").unwrap();
assert!(locate_wasmer(dir.path()).unwrap().ends_with(WASMER_BIN));
std::fs::write(dir.path().join(WASMER_BIN), b"stub").unwrap();
let found = locate_wasmer(dir.path()).unwrap();
assert_eq!(found, dir.path().join(WASMER_BIN));
}
#[test]
fn availability_missing_without_binary() {
if env_override(ENV_WASMER).is_some() {
return; }
let dir = tempfile::tempdir().unwrap();
let sb = WasmerSandbox::new(Some(dir.path().to_path_buf()));
assert!(matches!(
sb.availability(ru()),
SandboxAvailability::Missing(_)
));
}
#[test]
fn availability_ready_with_binary() {
if env_override(ENV_WASMER).is_some() {
return;
}
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join(WASMER_BIN), b"stub").unwrap();
let sb = WasmerSandbox::new(Some(dir.path().to_path_buf()));
assert_eq!(sb.availability(ru()), SandboxAvailability::Ready);
}
#[tokio::test]
async fn gate_rejects_second_concurrent_task() {
let dir = tempfile::tempdir().unwrap();
let sb = WasmerSandbox::new(Some(dir.path().to_path_buf()));
let _held = sb.gate.try_acquire().unwrap();
let err = sb
.run(
SandboxJob::new("print(1)", false, Duration::from_secs(5)),
ru(),
)
.await
.unwrap_err();
assert!(err.to_string().contains("занята"), "got: {err}");
}
#[tokio::test]
async fn busy_error_is_localized() {
let dir = tempfile::tempdir().unwrap();
let sb = WasmerSandbox::new(Some(dir.path().to_path_buf()));
let _held = sb.gate.try_acquire().unwrap();
let en = sb
.run(
SandboxJob::new("print(1)", false, Duration::from_secs(5)),
locale(Lang::En),
)
.await
.unwrap_err()
.to_string();
assert!(en.contains("busy"), "{en}");
assert!(!en.chars().any(|c| ('а'..='я').contains(&c)), "{en}");
}
#[tokio::test]
async fn gate_permit_released_after_run() {
let dir = tempfile::tempdir().unwrap();
let sb = WasmerSandbox::new(Some(dir.path().to_path_buf()));
if env_override(ENV_WASMER).is_some() {
return; }
let e1 = sb
.run(
SandboxJob::new("print(1)", false, Duration::from_secs(5)),
ru(),
)
.await
.unwrap_err();
assert!(e1.to_string().contains("wasmer"), "got: {e1}");
let e2 = sb
.run(
SandboxJob::new("print(1)", false, Duration::from_secs(5)),
ru(),
)
.await
.unwrap_err();
assert!(e2.to_string().contains("wasmer"), "got: {e2}");
}
#[test]
fn resolve_python_falls_back_to_registry_package() {
if env_override(ENV_PYTHON).is_some() {
return;
}
let dir = tempfile::tempdir().unwrap();
let sb = WasmerSandbox::new(Some(dir.path().to_path_buf()));
assert_eq!(sb.resolve_python(), OsString::from(DEFAULT_PYTHON_PKG));
}
fn sandbox_dir(site_packages: bool, image: bool) -> tempfile::TempDir {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join(WASMER_BIN), b"stub").unwrap();
if site_packages {
std::fs::create_dir(dir.path().join("site-packages")).unwrap();
}
if image {
std::fs::write(dir.path().join(SANDBOX_IMAGE), b"stub").unwrap();
}
dir
}
#[test]
fn a_stale_job_directory_is_swept_and_a_live_one_is_left() {
let temp = tempfile::tempdir().unwrap();
let ours = temp.path().join(format!("{JOB_PREFIX}0000"));
let theirs = temp.path().join("some-other-tool-42");
std::fs::create_dir_all(ours.join("in")).unwrap();
std::fs::write(ours.join("in").join("sales.csv"), b"month,total\n").unwrap();
std::fs::create_dir_all(&theirs).unwrap();
sweep_stale_in(temp.path(), Duration::from_secs(24 * 60 * 60));
assert!(
ours.is_dir(),
"a directory a running call may own must be left alone"
);
sweep_stale_in(temp.path(), Duration::ZERO);
assert!(
!ours.exists(),
"a stale job directory goes, and the chat's copies with it"
);
assert!(
theirs.is_dir(),
"another program's temp directory is not ours to delete"
);
}
#[test]
fn a_candidate_image_is_what_runs_when_one_is_named() {
let dir = sandbox_dir(true, true);
let candidate = format!("{SANDBOX_IMAGE}.partial");
std::fs::write(dir.path().join(&candidate), b"fresh").unwrap();
let installed = WasmerSandbox::new(Some(dir.path().to_path_buf()))
.plan()
.unwrap();
assert_eq!(installed.program, dir.path().join(SANDBOX_IMAGE));
let fresh = WasmerSandbox::for_candidate(dir.path().to_path_buf(), &candidate)
.plan()
.unwrap();
assert_eq!(fresh.program, dir.path().join(&candidate));
std::fs::remove_file(dir.path().join(&candidate)).unwrap();
let plan = WasmerSandbox::for_candidate(dir.path().to_path_buf(), &candidate).plan();
assert!(
plan.is_none(),
"a missing candidate must not fall through to the installed image: {plan:?}"
);
}
#[test]
fn the_packed_image_runs_and_nothing_is_mounted() {
let dir = sandbox_dir(true, true);
let plan = WasmerSandbox::new(Some(dir.path().to_path_buf()))
.plan()
.unwrap();
assert_eq!(
plan.program,
dir.path().join(SANDBOX_IMAGE).into_os_string()
);
assert_eq!(plan.site_mount, None);
assert!(plan.site_on_path);
}
#[tokio::test]
async fn unpacked_site_packages_is_refused_with_the_way_out() {
let dir = sandbox_dir(true, false);
let sb = WasmerSandbox::new(Some(dir.path().to_path_buf()));
assert_eq!(sb.plan(), None);
let SandboxAvailability::Missing(why) = sb.availability(locale(Lang::En)) else {
panic!("an unpacked site-packages must not be Ready");
};
assert!(why.contains("mindfork sandbox setup"), "{why}");
let err = sb
.run(
SandboxJob::new("print(1)", false, Duration::from_secs(5)),
locale(Lang::En),
)
.await
.unwrap_err();
assert!(err.to_string().contains("mindfork sandbox setup"), "{err}");
}
#[test]
fn provisioning_mounts_the_directory() {
let dir = sandbox_dir(true, true);
let plan = WasmerSandbox::for_provisioning(dir.path().to_path_buf())
.plan()
.unwrap();
assert_eq!(plan.site_mount, Some(dir.path().join("site-packages")));
assert!(plan.site_on_path);
assert_ne!(
plan.program,
dir.path().join(SANDBOX_IMAGE).into_os_string()
);
}
#[test]
fn without_packages_plain_python_runs() {
let dir = sandbox_dir(false, false);
let sb = WasmerSandbox::new(Some(dir.path().to_path_buf()));
let plan = sb.plan().unwrap();
assert_eq!(plan.program, sb.resolve_python());
assert_eq!(plan.site_mount, None);
assert!(!plan.site_on_path);
assert_eq!(sb.availability(ru()), SandboxAvailability::Ready);
}
}