use std::collections::BTreeMap;
use std::fmt;
use std::io::Read;
use std::path::{Component, Path, PathBuf};
use std::sync::{Arc, Mutex, MutexGuard};
use crate::validation::{MAX_FORMAT_ID_BYTES, valid_nonempty_text};
use crate::{Error, SourceId};
const MAX_REFERENCED_FILES: usize = 4_096;
const MAX_REFERENCED_BYTES: u64 = 64 << 20;
const MAX_REFERENCED_DEPTH: usize = 64;
const UTF8_BOM: [u8; 3] = [0xEF, 0xBB, 0xBF];
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct FormatId(Box<str>);
impl FormatId {
pub fn new(id: impl Into<String>) -> Result<Self, Error> {
let id = id.into();
if !valid_format_id(&id) {
return Err(Error::new(
&crate::codes::REQUEST_FORMAT_INVALID_ID,
"a format ID must be bounded lower case ASCII segments separated by single hyphens",
));
}
Ok(Self(id.into_boxed_str()))
}
#[must_use]
pub fn as_str(&self) -> &str {
&self.0
}
}
impl fmt::Display for FormatId {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(&self.0)
}
}
fn valid_format_id(id: &str) -> bool {
if id.is_empty() || id.len() > MAX_FORMAT_ID_BYTES {
return false;
}
let bytes = id.as_bytes();
if !bytes[0].is_ascii_lowercase() || bytes.last() == Some(&b'-') {
return false;
}
let mut previous_hyphen = false;
for byte in bytes {
if *byte == b'-' {
if previous_hyphen {
return false;
}
previous_hyphen = true;
} else if byte.is_ascii_lowercase() || byte.is_ascii_digit() {
previous_hyphen = false;
} else {
return false;
}
}
true
}
#[derive(Debug)]
struct SourceBufferData {
id: SourceId,
name: Box<str>,
bytes: Arc<[u8]>,
directory: Box<[Box<str>]>,
}
#[derive(Clone, Debug)]
pub struct SourceBuffer(Arc<SourceBufferData>);
impl SourceBuffer {
fn new(
id: SourceId,
name: impl Into<String>,
bytes: Arc<[u8]>,
directory: Vec<String>,
) -> Self {
Self(Arc::new(SourceBufferData {
id,
name: name.into().into_boxed_str(),
bytes,
directory: directory.into_iter().map(String::into_boxed_str).collect(),
}))
}
#[must_use]
pub fn id(&self) -> &SourceId {
&self.0.id
}
#[must_use]
pub fn name(&self) -> &str {
&self.0.name
}
#[must_use]
pub fn bytes(&self) -> &[u8] {
&self.0.bytes
}
#[must_use]
pub fn shared_bytes(&self) -> Arc<[u8]> {
Arc::clone(&self.0.bytes)
}
#[must_use]
pub fn has_utf8_bom(&self) -> bool {
self.0.bytes.starts_with(&UTF8_BOM)
}
#[must_use]
pub fn content_bytes(&self) -> &[u8] {
let bytes: &[u8] = &self.0.bytes;
if bytes.starts_with(&UTF8_BOM) {
&bytes[UTF8_BOM.len()..]
} else {
bytes
}
}
pub fn directory_segments(&self) -> impl Iterator<Item = &str> {
self.0.directory.iter().map(AsRef::as_ref)
}
}
#[derive(Debug)]
struct FileAcquisition {
root_display: PathBuf,
selected: bool,
state: Mutex<AcquisitionState>,
}
#[derive(Debug, Default)]
struct AcquisitionState {
root: Option<platform::RootHandle>,
cache: BTreeMap<String, SourceBuffer>,
listed: Option<Vec<crate::ArtifactPath>>,
files: usize,
bytes: u64,
}
impl AcquisitionState {
fn pinned_root(&mut self, root_display: &Path) -> Result<&platform::RootHandle, Error> {
if self.root.is_none() {
let root = platform::open_root(root_display)
.map_err(|cause| open_error(&crate::codes::READ_IO_OPEN, root_display, cause))?;
self.root = Some(root);
}
Ok(self.root.as_ref().expect("pinned above"))
}
}
#[derive(Debug)]
enum SourceProvider {
Memory {
primary: SourceBuffer,
named: BTreeMap<String, SourceBuffer>,
},
File {
primary: SourceBuffer,
acquisition: FileAcquisition,
},
Directory {
acquisition: FileAcquisition,
},
}
pub const PRIMARY_SOURCE_ID: &str = "/input";
#[derive(Clone)]
pub struct Source {
name: Arc<str>,
provider: Arc<SourceProvider>,
declared_format: Option<FormatId>,
}
impl Source {
pub fn open(path: impl Into<PathBuf>) -> Result<Self, Error> {
let path = path.into();
if path.as_os_str().is_empty() {
return Err(Error::new(
&crate::codes::REQUEST_SOURCE_INVALID_PATH,
"source path cannot be empty",
));
}
let name: Arc<str> = path.to_string_lossy().into_owned().into();
let file = match platform::open_no_follow(&path) {
Ok(file) => file,
Err(error) if platform::is_symlink_refusal(&error) => {
return Err(Error::new(
&crate::codes::REQUEST_SOURCE_SYMLINK_REFUSED,
format!("source `{}` is a symbolic link", path.display()),
));
}
Err(error) if platform::is_directory_open_failure(&error, &path) => {
return Self::open_directory(name, &path);
}
Err(cause) => return Err(open_error(&crate::codes::READ_IO_OPEN, &path, cause)),
};
let metadata = file
.metadata()
.map_err(|cause| open_error(&crate::codes::READ_IO_METADATA, &path, cause))?;
if metadata.is_dir() {
drop(file);
return Self::open_directory(name, &path);
}
let bytes = read_open_file(file, &name, u64::MAX)?;
let root_display = canonical_parent(&path)?;
let primary = SourceBuffer::new(
SourceId::new(PRIMARY_SOURCE_ID)?,
name.to_string(),
bytes,
Vec::new(),
);
Ok(Self {
name,
provider: Arc::new(SourceProvider::File {
primary,
acquisition: FileAcquisition {
root_display,
selected: false,
state: Mutex::new(AcquisitionState::default()),
},
}),
declared_format: None,
})
}
fn open_directory(name: Arc<str>, path: &Path) -> Result<Self, Error> {
let root_display = std::fs::canonicalize(path)
.map_err(|cause| open_error(&crate::codes::READ_IO_METADATA, path, cause))?;
Ok(Self {
name,
provider: Arc::new(SourceProvider::Directory {
acquisition: FileAcquisition {
root_display,
selected: false,
state: Mutex::new(AcquisitionState::default()),
},
}),
declared_format: None,
})
}
pub fn from_bytes(name: impl Into<String>, bytes: impl Into<Arc<[u8]>>) -> Result<Self, Error> {
let name = name.into();
if !valid_nonempty_text(&name) {
return Err(Error::new(
&crate::codes::REQUEST_SOURCE_INVALID_NAME,
"an in-memory source requires a nonempty bounded name",
));
}
let primary = SourceBuffer::new(
SourceId::new(PRIMARY_SOURCE_ID)?,
name.clone(),
bytes.into(),
Vec::new(),
);
Ok(Self {
name: name.into(),
provider: Arc::new(SourceProvider::Memory {
primary,
named: BTreeMap::new(),
}),
declared_format: None,
})
}
pub fn with_named_buffer(
self,
name: impl Into<String>,
bytes: impl Into<Arc<[u8]>>,
) -> Result<Self, Error> {
let name = name.into();
let segments = resolve_segments(&[], &name)?;
let key = segments.join("/");
let mut provider = Arc::try_unwrap(self.provider).map_err(|_| {
Error::new(
&crate::codes::REQUEST_SOURCE_INVALID_NAME,
"named buffers are supplied while constructing a source, before it is shared",
)
})?;
let SourceProvider::Memory { named, .. } = &mut provider else {
return Err(Error::new(
&crate::codes::REQUEST_SOURCE_INVALID_NAME,
"named buffers belong to in-memory sources; a file source acquires referenced files beneath its root",
));
};
let directory = segments[..segments.len() - 1].to_vec();
let buffer = SourceBuffer::new(SourceId::new(&key)?, key.clone(), bytes.into(), directory);
named.insert(key, buffer);
Ok(Self {
name: self.name,
provider: Arc::new(provider),
declared_format: self.declared_format,
})
}
pub fn with_acquisition_root(self, root: impl Into<PathBuf>) -> Result<Self, Error> {
let requested = root.into();
let SourceProvider::File {
primary,
acquisition,
} = &*self.provider
else {
return Err(Error::new(
&crate::codes::REQUEST_SOURCE_INVALID_PATH,
"an acquisition root applies to a file source",
));
};
let canonical = std::fs::canonicalize(&requested)
.map_err(|cause| open_error(&crate::codes::READ_IO_METADATA, &requested, cause))?;
let Ok(remainder) = acquisition.root_display.strip_prefix(&canonical) else {
return Err(Error::new(
&crate::codes::REQUEST_SOURCE_INVALID_PATH,
format!(
"the case file directory {} is outside the requested acquisition root {}",
acquisition.root_display.display(),
canonical.display()
),
));
};
let mut directory = Vec::new();
for component in remainder.components() {
let Component::Normal(segment) = component else {
return Err(Error::new(
&crate::codes::REQUEST_SOURCE_INVALID_PATH,
"the acquisition root does not resolve to a plain prefix of the case directory",
));
};
let Some(segment) = segment.to_str().filter(|text| plain_segment(text)) else {
return Err(Error::new(
&crate::codes::REQUEST_SOURCE_INVALID_PATH,
"the acquisition root does not resolve to a plain prefix of the case directory",
));
};
directory.push(segment.to_owned());
}
let primary = SourceBuffer::new(
primary.id().clone(),
primary.name().to_owned(),
primary.shared_bytes(),
directory,
);
Ok(Self {
name: self.name,
provider: Arc::new(SourceProvider::File {
primary,
acquisition: FileAcquisition {
root_display: canonical,
selected: true,
state: Mutex::new(AcquisitionState::default()),
},
}),
declared_format: self.declared_format,
})
}
#[must_use]
pub fn with_format(mut self, format: FormatId) -> Self {
self.declared_format = Some(format);
self
}
#[must_use]
pub fn name(&self) -> &str {
&self.name
}
#[must_use]
pub const fn format(&self) -> Option<&FormatId> {
self.declared_format.as_ref()
}
#[must_use]
pub fn is_directory(&self) -> bool {
matches!(&*self.provider, SourceProvider::Directory { .. })
}
pub fn primary_buffer(&self) -> Result<SourceBuffer, Error> {
match &*self.provider {
SourceProvider::Memory { primary, .. } | SourceProvider::File { primary, .. } => {
Ok(primary.clone())
}
SourceProvider::Directory { .. } => Err(Error::new(
&crate::codes::REQUEST_SOURCE_DIRECTORY_REQUIRED,
"a directory source has no implicit primary buffer",
)
.with_source(self.clone())),
}
}
pub fn buffer(&self, name: &crate::ArtifactPath) -> Result<SourceBuffer, Error> {
let SourceProvider::Directory { acquisition } = &*self.provider else {
return Err(Error::new(
&crate::codes::REQUEST_SOURCE_DIRECTORY_REQUIRED,
"named child buffers require a directory source",
)
.with_source(self.clone()));
};
let segments = resolve_segments(&[], name.as_str())
.map_err(|error| error.with_source(self.clone()))?;
acquisition
.acquire(&segments)
.map_err(|error| error.with_source(self.clone()))
}
pub fn root_buffer(&self, name: &str) -> Result<SourceBuffer, Error> {
match &*self.provider {
SourceProvider::Memory { named, .. } => {
let segments = resolve_segments(&[], name)?;
let key = segments.join("/");
named.get(&key).cloned().ok_or_else(|| {
Error::new(
&crate::codes::REQUEST_SOURCE_UNKNOWN_BUFFER,
format!(
"referenced buffer `{key}` was not supplied to this in-memory source"
),
)
.with_source(self.clone())
})
}
SourceProvider::File { acquisition, .. }
| SourceProvider::Directory { acquisition } => {
let segments = resolve_segments(&[], name)?;
acquisition
.acquire(&segments)
.map_err(|error| error.with_source(self.clone()))
}
}
}
#[must_use]
pub fn selected_acquisition_root(&self) -> Option<&Path> {
match &*self.provider {
SourceProvider::Memory { .. } | SourceProvider::Directory { .. } => None,
SourceProvider::File { acquisition, .. } => acquisition
.selected
.then_some(acquisition.root_display.as_path()),
}
}
pub fn referenced_buffer(
&self,
referrer: &SourceBuffer,
name: &str,
) -> Result<SourceBuffer, Error> {
let referrer_directory: Vec<&str> = referrer.directory_segments().collect();
match &*self.provider {
SourceProvider::Memory { named, .. } => {
let segments = resolve_segments(&referrer_directory, name)?;
let key = segments.join("/");
named.get(&key).cloned().ok_or_else(|| {
Error::new(
&crate::codes::REQUEST_SOURCE_UNKNOWN_BUFFER,
format!(
"referenced buffer `{key}` was not supplied to this in-memory source"
),
)
.with_source(self.clone())
})
}
SourceProvider::File { acquisition, .. }
| SourceProvider::Directory { acquisition } => {
let segments = match absolute_to_root_relative(&acquisition.root_display, name) {
Some(root_relative) => root_relative?,
None => resolve_segments(&referrer_directory, name)?,
};
acquisition
.acquire(&segments)
.map_err(|error| error.with_source(self.clone()))
}
}
}
#[allow(clippy::too_many_lines)] pub fn entry_names(&self) -> Result<Vec<crate::ArtifactPath>, Error> {
match &*self.provider {
SourceProvider::Memory { named, .. } => named
.keys()
.map(|name| crate::ArtifactPath::new(name.clone()))
.collect(),
SourceProvider::File { .. } => Err(Error::new(
&crate::codes::REQUEST_SOURCE_DIRECTORY_REQUIRED,
"entry listing requires a directory source",
)
.with_source(self.clone())),
SourceProvider::Directory { acquisition } => {
struct Frame<Handle> {
prefix: Vec<String>,
directory: Handle,
subdirectories: Vec<String>,
}
let mut state = acquisition.lock();
if let Some(listed) = &state.listed {
return Ok(listed.clone());
}
let root = state.pinned_root(&acquisition.root_display)?;
let budget_refusal = || {
Error::new(
&crate::codes::READ_IO_REFERENCE_BUDGET,
format!(
"the source directory holds more than {MAX_REFERENCED_FILES} entries"
),
)
};
let root_handle = root.duplicate_handle().map_err(|cause| {
Error::new(
&crate::codes::READ_IO_METADATA,
"cannot list the source directory root",
)
.with_cause(cause)
})?;
let mut names = Vec::new();
let mut undescended = 0usize;
let mut frames = Vec::new();
let mut arriving = Some((Vec::<String>::new(), root_handle));
while let Some((prefix, directory)) = arriving.take() {
let allowance = MAX_REFERENCED_FILES
.checked_sub(names.len() + undescended)
.filter(|allowance| *allowance > 0)
.ok_or_else(budget_refusal)?;
let entries =
platform::list_entries(&directory, allowance).map_err(|cause| {
if platform::is_entry_budget(&cause) {
budget_refusal()
} else {
listing_error(&prefix.join("/"), cause)
}
})?;
let mut subdirectories = Vec::new();
for (name, is_directory) in entries {
if names.len() + undescended + subdirectories.len() >= MAX_REFERENCED_FILES
{
return Err(budget_refusal());
}
if is_directory {
if prefix.len() + 1 >= MAX_REFERENCED_DEPTH {
return Err(Error::new(
&crate::codes::READ_IO_REFERENCE_BUDGET,
format!(
"the source directory nests more than {MAX_REFERENCED_DEPTH} levels deep"
),
));
}
subdirectories.push(name);
} else {
let mut child = prefix.clone();
child.push(name);
names.push(crate::ArtifactPath::new(child.join("/"))?);
}
}
undescended += subdirectories.len();
frames.push(Frame {
prefix,
directory,
subdirectories,
});
while let Some(frame) = frames.last_mut() {
let Some(name) = frame.subdirectories.pop() else {
frames.pop();
continue;
};
undescended -= 1;
let mut child = frame.prefix.clone();
let handle = platform::open_child_directory(&frame.directory, &name)
.map_err(|cause| {
child.push(name.clone());
listing_error(&child.join("/"), cause)
})?;
child.push(name);
arriving = Some((child, handle));
break;
}
}
names.sort();
state.listed = Some(names.clone());
Ok(names)
}
}
}
#[must_use]
pub fn acquired_buffers(&self) -> Vec<SourceBuffer> {
match &*self.provider {
SourceProvider::Memory { primary, named } => {
let mut buffers = vec![primary.clone()];
buffers.extend(named.values().cloned());
buffers
}
SourceProvider::File {
primary,
acquisition,
} => {
let mut buffers = vec![primary.clone()];
buffers.extend(acquisition.lock().cache.values().cloned());
buffers
}
SourceProvider::Directory { acquisition } => {
acquisition.lock().cache.values().cloned().collect()
}
}
}
}
impl fmt::Debug for Source {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("Source")
.field("name", &self.name)
.field("is_directory", &self.is_directory())
.field("declared_format", &self.declared_format)
.field("acquired_buffer_count", &self.acquired_buffers().len())
.finish_non_exhaustive()
}
}
impl FileAcquisition {
fn lock(&self) -> MutexGuard<'_, AcquisitionState> {
self.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn acquire(&self, segments: &[String]) -> Result<SourceBuffer, Error> {
let key = segments.join("/");
let mut state = self.lock();
if let Some(buffer) = state.cache.get(&key) {
return Ok(buffer.clone());
}
if state.files >= MAX_REFERENCED_FILES {
return Err(Error::new(
&crate::codes::READ_IO_REFERENCE_BUDGET,
format!("this source already acquired {MAX_REFERENCED_FILES} referenced files"),
));
}
let remaining = MAX_REFERENCED_BYTES.saturating_sub(state.bytes);
let root = state.pinned_root(&self.root_display)?;
let file = root.open_beneath(segments).map_err(|error| {
if platform::is_symlink_refusal(&error) {
Error::new(
&crate::codes::REQUEST_SOURCE_SYMLINK_REFUSED,
format!("referenced file `{key}` crosses a symbolic link"),
)
} else {
Error::new(
&crate::codes::READ_IO_OPEN,
format!("cannot open referenced file `{key}`"),
)
.with_cause(error)
}
})?;
let bytes = read_open_file(file, &key, remaining)?;
let directory = segments[..segments.len() - 1].to_vec();
let buffer = SourceBuffer::new(SourceId::new(&key)?, key.clone(), bytes, directory);
state.files += 1;
state.bytes += buffer.bytes().len() as u64;
state.cache.insert(key, buffer.clone());
Ok(buffer)
}
}
fn plain_segment(segment: &str) -> bool {
if segment.is_empty()
|| segment == "."
|| segment == ".."
|| segment.contains(['/', '\\', '\0', ':'])
{
return false;
}
let mut components = Path::new(segment).components();
matches!(components.next(), Some(Component::Normal(text)) if text == std::ffi::OsStr::new(segment))
&& components.next().is_none()
}
fn resolve_segments(referrer_directory: &[&str], name: &str) -> Result<Vec<String>, Error> {
if name.is_empty()
|| name.len() > crate::validation::MAX_ARTIFACT_PATH_BYTES
|| name.contains('\0')
|| name.starts_with(['/', '\\'])
{
return Err(Error::new(
&crate::codes::REQUEST_SOURCE_INVALID_PATH,
"a referenced name must be a nonempty bounded relative path",
));
}
let mut segments: Vec<String> = referrer_directory
.iter()
.map(|segment| (*segment).to_owned())
.collect();
for raw in name.split(['/', '\\']) {
match raw {
"" | "." => {}
".." => {
if segments.pop().is_none() {
return Err(Error::new(
&crate::codes::REQUEST_SOURCE_ESCAPES_ROOT,
format!("referenced name `{name}` resolves outside the acquisition root"),
));
}
}
segment => {
if segment.len() > crate::validation::MAX_ARTIFACT_PATH_BYTES
|| !plain_segment(segment)
{
return Err(Error::new(
&crate::codes::REQUEST_SOURCE_INVALID_PATH,
format!("referenced name `{name}` is not a portable relative path"),
));
}
segments.push(segment.to_owned());
}
}
}
if segments.is_empty() {
return Err(Error::new(
&crate::codes::REQUEST_SOURCE_INVALID_PATH,
format!("referenced name `{name}` does not name a file"),
));
}
if segments.len() > MAX_REFERENCED_DEPTH {
return Err(Error::new(
&crate::codes::READ_IO_REFERENCE_BUDGET,
format!("referenced name `{name}` nests more than {MAX_REFERENCED_DEPTH} levels deep"),
));
}
Ok(segments)
}
fn absolute_to_root_relative(root: &Path, name: &str) -> Option<Result<Vec<String>, Error>> {
let path = Path::new(name);
if !path.is_absolute() {
return None;
}
let Ok(remainder) = path.strip_prefix(root) else {
return Some(Err(Error::new(
&crate::codes::REQUEST_SOURCE_ESCAPES_ROOT,
format!("referenced name `{name}` resolves outside the acquisition root"),
)));
};
let mut segments = Vec::new();
for component in remainder.components() {
let text = match component {
Component::Normal(text) => text.to_str(),
_ => None,
};
let Some(text) = text.filter(|text| plain_segment(text)) else {
return Some(Err(Error::new(
&crate::codes::REQUEST_SOURCE_INVALID_PATH,
format!("referenced name `{name}` is not a portable path beneath the root"),
)));
};
segments.push(text.to_owned());
}
if segments.is_empty() {
return Some(Err(Error::new(
&crate::codes::REQUEST_SOURCE_INVALID_PATH,
format!("referenced name `{name}` does not name a file"),
)));
}
if segments.len() > MAX_REFERENCED_DEPTH {
return Some(Err(Error::new(
&crate::codes::READ_IO_REFERENCE_BUDGET,
format!("referenced name `{name}` nests more than {MAX_REFERENCED_DEPTH} levels deep"),
)));
}
Some(Ok(segments))
}
fn canonical_parent(path: &Path) -> Result<PathBuf, Error> {
let parent = match path.parent() {
Some(parent) if !parent.as_os_str().is_empty() => parent,
_ => Path::new("."),
};
std::fs::canonicalize(parent)
.map_err(|cause| open_error(&crate::codes::READ_IO_METADATA, parent, cause))
}
fn open_error(info: &'static crate::DiagnosticInfo, path: &Path, cause: std::io::Error) -> Error {
Error::new(info, format!("cannot open source `{}`", path.display())).with_cause(cause)
}
fn listing_error(display: &str, cause: std::io::Error) -> Error {
if platform::is_symlink_refusal(&cause) {
Error::new(
&crate::codes::REQUEST_SOURCE_SYMLINK_REFUSED,
format!("source directory `{display}` crosses a symbolic link"),
)
} else {
Error::new(
&crate::codes::READ_IO_METADATA,
format!("cannot list source directory `{display}`"),
)
.with_cause(cause)
}
}
fn read_open_file(file: std::fs::File, name: &str, max_bytes: u64) -> Result<Arc<[u8]>, Error> {
let metadata = file.metadata().map_err(|cause| {
Error::new(
&crate::codes::READ_IO_METADATA,
format!("cannot inspect source buffer `{name}`"),
)
.with_cause(cause)
})?;
if !metadata.is_file() {
return Err(Error::new(
&crate::codes::REQUEST_SOURCE_NOT_A_FILE,
format!("source buffer `{name}` is not a regular file"),
));
}
let declared_length = metadata.len();
if declared_length > max_bytes {
return Err(Error::new(
&crate::codes::READ_IO_REFERENCE_BUDGET,
format!(
"referenced file `{name}` would take this source past its {MAX_REFERENCED_BYTES} byte acquisition budget"
),
));
}
let capacity = usize::try_from(declared_length).map_err(|cause| {
Error::new(
&crate::codes::READ_IO_ALLOCATION_REFUSED,
format!("source buffer `{name}` is too large for this platform"),
)
.with_cause(cause)
})?;
let mut bytes = Vec::new();
bytes.try_reserve_exact(capacity).map_err(|cause| {
Error::new(
&crate::codes::READ_IO_ALLOCATION_REFUSED,
format!("cannot reserve {declared_length} bytes for source buffer `{name}`"),
)
.with_cause(cause)
})?;
let read_limit = declared_length
.checked_add(1)
.ok_or_else(|| {
Error::new(
&crate::codes::READ_IO_ALLOCATION_REFUSED,
format!("source buffer `{name}` is too large to read safely"),
)
})?
.min(max_bytes.saturating_add(1));
let mut file = file;
file.by_ref()
.take(read_limit)
.read_to_end(&mut bytes)
.map_err(|cause| {
Error::new(
&crate::codes::READ_IO_READ,
format!("cannot read source buffer `{name}`"),
)
.with_cause(cause)
})?;
if bytes.len() != capacity {
return Err(Error::new(
&crate::codes::READ_IO_SOURCE_CHANGED,
format!("source buffer `{name}` changed length while it was read"),
));
}
Ok(bytes.into())
}
#[cfg(unix)]
mod platform {
use std::ffi::CString;
use std::fs::File;
use std::os::fd::{AsRawFd, FromRawFd, OwnedFd};
use std::os::unix::ffi::OsStrExt;
use std::path::Path;
#[derive(Debug)]
pub(super) struct RootHandle(OwnedFd);
pub(super) fn open_no_follow(path: &Path) -> std::io::Result<File> {
let path = c_string(path.as_os_str().as_bytes())?;
let fd = unsafe {
libc::open(
path.as_ptr(),
libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC | libc::O_NONBLOCK,
)
};
if fd < 0 {
return Err(std::io::Error::last_os_error());
}
let file = unsafe { File::from_raw_fd(fd) };
clear_nonblock(&file)?;
Ok(file)
}
pub(super) fn open_root(path: &Path) -> std::io::Result<RootHandle> {
let path = c_string(path.as_os_str().as_bytes())?;
let fd = unsafe {
libc::open(
path.as_ptr(),
libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC | libc::O_NONBLOCK,
)
};
if fd < 0 {
return Err(std::io::Error::last_os_error());
}
let file = unsafe { File::from_raw_fd(fd) };
if !file.metadata()?.is_dir() {
return Err(std::io::Error::from(std::io::ErrorKind::NotADirectory));
}
clear_nonblock(&file)?;
Ok(RootHandle(file.into()))
}
impl RootHandle {
pub(super) fn open_beneath(&self, segments: &[String]) -> std::io::Result<File> {
let mut directory: Option<OwnedFd> = None;
let (file_segment, directories) =
segments.split_last().expect("resolution yields a file");
for segment in directories {
let next = self.open_at(
directory.as_ref(),
segment,
libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC | libc::O_NONBLOCK,
)?;
let next = File::from(next);
if !next.metadata()?.is_dir() {
return Err(std::io::Error::from(std::io::ErrorKind::NotADirectory));
}
directory = Some(next.into());
}
let fd = self.open_at(
directory.as_ref(),
file_segment,
libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC | libc::O_NONBLOCK,
)?;
let file = File::from(fd);
clear_nonblock(&file)?;
Ok(file)
}
pub(super) fn duplicate_handle(&self) -> std::io::Result<DirectoryHandle> {
let segment = c_string(b".")?;
let fd = unsafe {
libc::openat(
self.0.as_raw_fd(),
segment.as_ptr(),
libc::O_RDONLY | libc::O_CLOEXEC | libc::O_DIRECTORY,
)
};
if fd < 0 {
return Err(std::io::Error::last_os_error());
}
Ok(unsafe { OwnedFd::from_raw_fd(fd) })
}
fn open_at(
&self,
directory: Option<&OwnedFd>,
segment: &str,
flags: libc::c_int,
) -> std::io::Result<OwnedFd> {
if !super::plain_segment(segment) {
return Err(std::io::Error::from(std::io::ErrorKind::InvalidInput));
}
let at = directory.map_or_else(|| self.0.as_raw_fd(), AsRawFd::as_raw_fd);
let segment = c_string(segment.as_bytes())?;
let fd = unsafe { libc::openat(at, segment.as_ptr(), flags) };
if fd < 0 {
return Err(std::io::Error::last_os_error());
}
Ok(unsafe { OwnedFd::from_raw_fd(fd) })
}
}
pub(super) type DirectoryHandle = OwnedFd;
pub(super) fn open_child_directory(
parent: &DirectoryHandle,
name: &str,
) -> std::io::Result<DirectoryHandle> {
if !super::plain_segment(name) {
return Err(std::io::Error::from(std::io::ErrorKind::InvalidInput));
}
let segment = c_string(name.as_bytes())?;
let fd = unsafe {
libc::openat(
parent.as_raw_fd(),
segment.as_ptr(),
libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC | libc::O_NONBLOCK,
)
};
if fd < 0 {
return Err(std::io::Error::last_os_error());
}
let file = unsafe { File::from_raw_fd(fd) };
if !file.metadata()?.is_dir() {
return Err(std::io::Error::from(std::io::ErrorKind::NotADirectory));
}
Ok(file.into())
}
const ENTRY_BUDGET_MARKER: &str = "directory entry allowance exhausted";
fn entry_budget_error() -> std::io::Error {
std::io::Error::other(ENTRY_BUDGET_MARKER)
}
pub(super) fn is_entry_budget(error: &std::io::Error) -> bool {
error.kind() == std::io::ErrorKind::Other && error.to_string().contains(ENTRY_BUDGET_MARKER)
}
pub(super) fn list_entries(
directory: &DirectoryHandle,
max: usize,
) -> std::io::Result<Vec<(String, bool)>> {
use std::os::fd::IntoRawFd;
let raw = directory.try_clone()?.into_raw_fd();
let stream = unsafe { libc::fdopendir(raw) };
if stream.is_null() {
let error = std::io::Error::last_os_error();
unsafe { libc::close(raw) };
return Err(error);
}
let mut entries = Vec::new();
loop {
errno_clear();
let entry = unsafe { libc::readdir(stream) };
if entry.is_null() {
let error = std::io::Error::last_os_error();
unsafe { libc::closedir(stream) };
if error.raw_os_error().is_some_and(|code| code != 0) {
return Err(error);
}
break;
}
let name_bytes = unsafe {
std::ffi::CStr::from_ptr((&raw const (*entry).d_name).cast::<libc::c_char>())
};
let Ok(name) = name_bytes.to_str() else {
continue;
};
if name == "." || name == ".." {
continue;
}
if entries.len() == max {
unsafe { libc::closedir(stream) };
return Err(entry_budget_error());
}
let kind = unsafe { (*entry).d_type };
let is_directory = match kind {
libc::DT_DIR => true,
libc::DT_UNKNOWN => {
let mut stat: libc::stat = unsafe { std::mem::zeroed() };
let segment =
c_string(name.as_bytes()).expect("directory entry names carry no NUL");
let status = unsafe {
libc::fstatat(
libc::dirfd(stream),
segment.as_ptr(),
&raw mut stat,
libc::AT_SYMLINK_NOFOLLOW,
)
};
status == 0 && stat.st_mode & libc::S_IFMT == libc::S_IFDIR
}
_ => false,
};
entries.push((name.to_owned(), is_directory));
}
Ok(entries)
}
fn errno_clear() {
unsafe {
*errno_location() = 0;
}
}
#[cfg(target_os = "macos")]
fn errno_location() -> *mut libc::c_int {
unsafe { libc::__error() }
}
#[cfg(not(target_os = "macos"))]
fn errno_location() -> *mut libc::c_int {
unsafe { libc::__errno_location() }
}
fn clear_nonblock(file: &File) -> std::io::Result<()> {
let fd = file.as_raw_fd();
let flags = unsafe { libc::fcntl(fd, libc::F_GETFL) };
if flags < 0 {
return Err(std::io::Error::last_os_error());
}
let status = unsafe { libc::fcntl(fd, libc::F_SETFL, flags & !libc::O_NONBLOCK) };
if status < 0 {
return Err(std::io::Error::last_os_error());
}
Ok(())
}
pub(super) fn is_symlink_refusal(error: &std::io::Error) -> bool {
matches!(error.raw_os_error(), Some(libc::ELOOP | libc::EMLINK))
}
pub(super) fn is_directory_open_failure(error: &std::io::Error, path: &Path) -> bool {
let _ = path;
error.raw_os_error() == Some(libc::EISDIR)
}
fn c_string(bytes: &[u8]) -> std::io::Result<CString> {
CString::new(bytes).map_err(|_| std::io::Error::from(std::io::ErrorKind::InvalidInput))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn open_root_refuses_a_symbolic_link_to_a_directory() {
let base = std::env::temp_dir().join(format!(
"powerio-core-open-root-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&base).unwrap();
assert!(open_root(&base).is_ok());
let link = base.join("link");
std::os::unix::fs::symlink(&base, &link).unwrap();
let error = open_root(&link).unwrap_err();
assert!(
is_symlink_refusal(&error) || error.kind() == std::io::ErrorKind::NotADirectory,
"{error:?}"
);
std::fs::remove_dir_all(&base).unwrap();
}
}
}
#[cfg(not(unix))]
mod platform {
use std::fs::File;
use std::path::{Path, PathBuf};
#[derive(Debug)]
pub(super) struct RootHandle {
root: PathBuf,
}
pub(super) fn open_no_follow(path: &Path) -> std::io::Result<File> {
let file = open_reparse_refused(path)?;
Ok(file)
}
pub(super) fn open_root(path: &Path) -> std::io::Result<RootHandle> {
drop(open_directory_pinned(path)?);
Ok(RootHandle {
root: path.to_path_buf(),
})
}
impl RootHandle {
pub(super) fn open_beneath(&self, segments: &[String]) -> std::io::Result<File> {
let mut path = self.root.clone();
let (file_segment, directories) =
segments.split_last().expect("resolution yields a file");
let mut held = Vec::with_capacity(directories.len() + 1);
held.push(open_directory_pinned(&self.root)?);
for segment in directories {
push_plain_segment(&mut path, segment)?;
held.push(open_directory_pinned(&path)?);
}
push_plain_segment(&mut path, file_segment)?;
let file = open_reparse_refused(&path)?;
drop(held);
Ok(file)
}
pub(super) fn duplicate_handle(&self) -> std::io::Result<DirectoryHandle> {
let handle = open_directory_pinned(&self.root)?;
Ok(DirectoryHandle {
path: self.root.clone(),
_handle: handle,
})
}
}
#[derive(Debug)]
pub(super) struct DirectoryHandle {
path: PathBuf,
_handle: File,
}
pub(super) fn open_child_directory(
parent: &DirectoryHandle,
name: &str,
) -> std::io::Result<DirectoryHandle> {
let mut path = parent.path.clone();
push_plain_segment(&mut path, name)?;
let handle = open_directory_pinned(&path)?;
Ok(DirectoryHandle {
path,
_handle: handle,
})
}
const ENTRY_BUDGET_MARKER: &str = "directory entry allowance exhausted";
fn entry_budget_error() -> std::io::Error {
std::io::Error::other(ENTRY_BUDGET_MARKER)
}
pub(super) fn is_entry_budget(error: &std::io::Error) -> bool {
error.kind() == std::io::ErrorKind::Other && error.to_string().contains(ENTRY_BUDGET_MARKER)
}
pub(super) fn list_entries(
directory: &DirectoryHandle,
max: usize,
) -> std::io::Result<Vec<(String, bool)>> {
let mut entries = Vec::new();
for entry in std::fs::read_dir(&directory.path)? {
let entry = entry?;
let Ok(name) = entry.file_name().into_string() else {
continue;
};
if entries.len() == max {
return Err(entry_budget_error());
}
let is_directory = entry.file_type()?.is_dir();
entries.push((name, is_directory));
}
Ok(entries)
}
fn push_plain_segment(path: &mut PathBuf, segment: &str) -> std::io::Result<()> {
if !super::plain_segment(segment) {
return Err(std::io::Error::from(std::io::ErrorKind::InvalidInput));
}
path.push(segment);
Ok(())
}
#[cfg(windows)]
fn open_directory_pinned(path: &Path) -> std::io::Result<File> {
use std::os::windows::fs::{MetadataExt, OpenOptionsExt};
const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000;
const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000;
const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x0000_0400;
const FILE_ATTRIBUTE_DIRECTORY: u32 = 0x0000_0010;
const FILE_SHARE_READ: u32 = 0x0000_0001;
const FILE_SHARE_WRITE: u32 = 0x0000_0002;
let file = std::fs::OpenOptions::new()
.read(true)
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE)
.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS)
.open(path)?;
let attributes = file.metadata()?.file_attributes();
if attributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 {
return Err(symlink_error());
}
if attributes & FILE_ATTRIBUTE_DIRECTORY == 0 {
return Err(std::io::Error::from(std::io::ErrorKind::NotADirectory));
}
Ok(file)
}
#[cfg(not(windows))]
fn open_directory_pinned(path: &Path) -> std::io::Result<File> {
let metadata = std::fs::symlink_metadata(path)?;
if metadata.file_type().is_symlink() {
return Err(symlink_error());
}
if !metadata.is_dir() {
return Err(std::io::Error::from(std::io::ErrorKind::NotADirectory));
}
File::open(path)
}
#[cfg(windows)]
fn open_reparse_refused(path: &Path) -> std::io::Result<File> {
use std::os::windows::fs::{MetadataExt, OpenOptionsExt};
const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000;
const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x0000_0400;
let file = std::fs::OpenOptions::new()
.read(true)
.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT)
.open(path)?;
let attributes = file.metadata()?.file_attributes();
if attributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 {
return Err(symlink_error());
}
Ok(file)
}
#[cfg(not(windows))]
fn open_reparse_refused(path: &Path) -> std::io::Result<File> {
let metadata = std::fs::symlink_metadata(path)?;
if metadata.file_type().is_symlink() {
return Err(symlink_error());
}
File::open(path)
}
fn symlink_error() -> std::io::Error {
std::io::Error::new(std::io::ErrorKind::InvalidData, "symbolic link refused")
}
pub(super) fn is_symlink_refusal(error: &std::io::Error) -> bool {
error.kind() == std::io::ErrorKind::InvalidData
&& error.to_string().contains("symbolic link refused")
}
pub(super) fn is_directory_open_failure(error: &std::io::Error, path: &Path) -> bool {
let _ = error;
std::fs::symlink_metadata(path).is_ok_and(|metadata| metadata.is_dir())
}
}
#[cfg(test)]
mod tests {
use std::time::{SystemTime, UNIX_EPOCH};
use super::*;
use crate::ArtifactPath;
static PROCESS_RESOURCE_TESTS: Mutex<()> = Mutex::new(());
fn process_resource_guard() -> MutexGuard<'static, ()> {
PROCESS_RESOURCE_TESTS
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
struct CountingAllocator;
static ALLOCATED_BYTES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
thread_local! {
static MEASURING: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
}
fn measured_bytes<T>(work: impl FnOnce() -> T) -> (T, usize) {
let _ = MEASURING.try_with(|flag| flag.set(true));
let before = ALLOCATED_BYTES.load(std::sync::atomic::Ordering::Relaxed);
let value = work();
let after = ALLOCATED_BYTES.load(std::sync::atomic::Ordering::Relaxed);
let _ = MEASURING.try_with(|flag| flag.set(false));
(value, after.saturating_sub(before))
}
unsafe impl std::alloc::GlobalAlloc for CountingAllocator {
unsafe fn alloc(&self, layout: std::alloc::Layout) -> *mut u8 {
if MEASURING.try_with(std::cell::Cell::get).unwrap_or(false) {
ALLOCATED_BYTES.fetch_add(layout.size(), std::sync::atomic::Ordering::Relaxed);
}
unsafe { std::alloc::System.alloc(layout) }
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: std::alloc::Layout) {
unsafe { std::alloc::System.dealloc(ptr, layout) }
}
unsafe fn realloc(
&self,
ptr: *mut u8,
layout: std::alloc::Layout,
new_size: usize,
) -> *mut u8 {
if MEASURING.try_with(std::cell::Cell::get).unwrap_or(false) {
ALLOCATED_BYTES.fetch_add(new_size, std::sync::atomic::Ordering::Relaxed);
}
unsafe { std::alloc::System.realloc(ptr, layout, new_size) }
}
}
#[global_allocator]
static COUNTING: CountingAllocator = CountingAllocator;
fn test_root(name: &str) -> PathBuf {
let nonce = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
std::env::temp_dir().join(format!(
"powerio-core-{name}-{}-{nonce}",
std::process::id()
))
}
#[test]
fn format_ids_use_the_exact_open_grammar() {
for id in ["matpower", "psse-raw", "doe-go-3", "x1"] {
assert_eq!(FormatId::new(id).unwrap().as_str(), id);
}
for id in ["", "1matpower", "MATPOWER", "psse--raw", "psse-", "p_sse"] {
assert!(FormatId::new(id).is_err(), "{id}");
}
assert!(FormatId::new("a".repeat(MAX_FORMAT_ID_BYTES + 1)).is_err());
}
#[test]
fn memory_sources_retain_arbitrary_binary_bytes_without_copying() {
let bytes: Arc<[u8]> = vec![0, 255, 0, 128].into();
let pointer = bytes.as_ptr();
let source = Source::from_bytes("input.bin", Arc::clone(&bytes))
.unwrap()
.with_format(FormatId::new("pwb").unwrap());
let buffer = source.primary_buffer().unwrap();
assert_eq!(buffer.bytes(), [0, 255, 0, 128]);
assert_eq!(buffer.bytes().as_ptr(), pointer);
assert_eq!(source.format().unwrap().as_str(), "pwb");
assert!(Source::from_bytes("", Vec::new()).is_err());
assert!(Source::from_bytes("x\0y", Vec::new()).is_err());
}
#[test]
fn a_bom_is_retained_and_skipped_for_the_parser_without_a_second_buffer() {
let bytes: Vec<u8> = [0xEF, 0xBB, 0xBF, b'm', b'p', b'c'].to_vec();
let source = Source::from_bytes("case.m", bytes).unwrap();
let buffer = source.primary_buffer().unwrap();
assert!(buffer.has_utf8_bom());
assert_eq!(buffer.bytes().len(), 6);
assert_eq!(buffer.content_bytes(), b"mpc");
assert_eq!(
buffer.content_bytes().as_ptr(),
buffer.bytes()[3..].as_ptr()
);
let plain = Source::from_bytes("case.m", b"mpc".to_vec()).unwrap();
let plain = plain.primary_buffer().unwrap();
assert!(!plain.has_utf8_bom());
assert_eq!(plain.content_bytes(), plain.bytes());
}
#[test]
fn a_memory_source_resolves_named_buffers_and_never_the_filesystem() {
let source = Source::from_bytes("master.dss", b"redirect sub/feeder.dss".to_vec())
.unwrap()
.with_named_buffer("sub/feeder.dss", b"feeder".to_vec())
.unwrap();
let primary = source.primary_buffer().unwrap();
let feeder = source
.referenced_buffer(&primary, "sub/feeder.dss")
.unwrap();
assert_eq!(feeder.bytes(), b"feeder");
let sibling = source
.referenced_buffer(&feeder, "../sub/feeder.dss")
.unwrap();
assert_eq!(sibling.bytes(), b"feeder");
assert!(source.referenced_buffer(&primary, "missing.dss").is_err());
let escape = source.referenced_buffer(&primary, "../outside.dss");
assert_eq!(
escape.unwrap_err().category(),
crate::ErrorCategory::Request
);
}
#[test]
fn a_file_source_acquires_referenced_files_beneath_its_containing_directory() {
let root = test_root("file-refs");
std::fs::create_dir_all(root.join("sub")).unwrap();
std::fs::write(root.join("master.dss"), b"master").unwrap();
std::fs::write(root.join("sub/feeder.dss"), b"feeder").unwrap();
std::fs::write(root.join("sub/coords.csv"), b"coords").unwrap();
let source = Source::open(root.join("master.dss")).unwrap();
let primary = source.primary_buffer().unwrap();
assert_eq!(primary.bytes(), b"master");
let feeder = source
.referenced_buffer(&primary, "sub/feeder.dss")
.unwrap();
assert_eq!(feeder.bytes(), b"feeder");
let coords = source.referenced_buffer(&feeder, "coords.csv").unwrap();
assert_eq!(coords.bytes(), b"coords");
let master_again = source.referenced_buffer(&feeder, "../master.dss").unwrap();
assert_eq!(master_again.bytes(), b"master");
let escape = source.referenced_buffer(&primary, "../escape.dss");
assert_eq!(
escape.unwrap_err().category(),
crate::ErrorCategory::Request
);
let again = source
.referenced_buffer(&primary, "sub/feeder.dss")
.unwrap();
assert_eq!(again.bytes().as_ptr(), feeder.bytes().as_ptr());
assert_eq!(source.acquired_buffers().len(), 4);
std::fs::remove_dir_all(root).unwrap();
}
#[test]
fn an_explicitly_wider_root_admits_shared_files_and_still_confines() {
let root = test_root("wider-root");
std::fs::create_dir_all(root.join("cases")).unwrap();
std::fs::create_dir_all(root.join("shared")).unwrap();
std::fs::write(root.join("cases/master.dss"), b"master").unwrap();
std::fs::write(root.join("shared/wires.dss"), b"wires").unwrap();
let narrow = Source::open(root.join("cases/master.dss")).unwrap();
let primary = narrow.primary_buffer().unwrap();
assert!(
narrow
.referenced_buffer(&primary, "../shared/wires.dss")
.is_err()
);
let wide = Source::open(root.join("cases/master.dss"))
.unwrap()
.with_acquisition_root(&root)
.unwrap();
let primary = wide.primary_buffer().unwrap();
let wires = wide
.referenced_buffer(&primary, "../shared/wires.dss")
.unwrap();
assert_eq!(wires.bytes(), b"wires");
assert!(
wide.referenced_buffer(&primary, "../../etc/passwd")
.is_err()
);
let outside = test_root("wider-root-outside");
std::fs::create_dir_all(&outside).unwrap();
assert!(
Source::open(root.join("cases/master.dss"))
.unwrap()
.with_acquisition_root(&outside)
.is_err()
);
std::fs::remove_dir_all(outside).ok();
std::fs::remove_dir_all(root).unwrap();
}
#[test]
fn directory_buffers_are_lazy_cached_and_binary_safe() {
let root = test_root("directory");
std::fs::create_dir_all(root.join("nested")).unwrap();
std::fs::write(root.join("nested/data.bin"), [0, 255, 7]).unwrap();
let source = Source::open(&root).unwrap();
assert!(source.is_directory());
assert!(source.acquired_buffers().is_empty());
let name = ArtifactPath::new("nested/data.bin").unwrap();
let first = source.buffer(&name).unwrap();
let second = source.buffer(&name).unwrap();
assert_eq!(first.bytes(), [0, 255, 7]);
assert_eq!(first.bytes().as_ptr(), second.bytes().as_ptr());
assert_eq!(source.acquired_buffers().len(), 1);
std::fs::remove_dir_all(root).unwrap();
}
#[cfg(unix)]
#[test]
fn source_acquisition_refuses_root_and_child_symlinks() {
use std::os::unix::fs::symlink;
let root = test_root("symlink");
std::fs::create_dir_all(&root).unwrap();
std::fs::write(root.join("real.bin"), b"real").unwrap();
symlink(root.join("real.bin"), root.join("link.bin")).unwrap();
let source = Source::open(&root).unwrap();
let error = source
.buffer(&ArtifactPath::new("link.bin").unwrap())
.unwrap_err();
assert_eq!(error.category(), crate::ErrorCategory::Request);
std::fs::create_dir_all(root.join("real-dir")).unwrap();
std::fs::write(root.join("real-dir/inner.bin"), b"inner").unwrap();
symlink(root.join("real-dir"), root.join("link-dir")).unwrap();
let error = source
.buffer(&ArtifactPath::new("link-dir/inner.bin").unwrap())
.unwrap_err();
assert_eq!(error.category(), crate::ErrorCategory::Request);
let root_link = root.with_extension("link");
symlink(&root, &root_link).unwrap();
assert!(Source::open(&root_link).is_err());
std::fs::remove_file(root_link).unwrap();
std::fs::remove_dir_all(root).unwrap();
}
#[cfg(unix)]
#[test]
fn a_named_pipe_is_refused_promptly_and_siblings_still_acquire() {
use std::os::unix::ffi::OsStrExt;
let root = test_root("fifo");
std::fs::create_dir_all(&root).unwrap();
std::fs::write(root.join("real.csv"), b"real").unwrap();
let fifo = root.join("pipe.dat");
let c_path = std::ffi::CString::new(fifo.as_os_str().as_bytes()).unwrap();
assert_eq!(unsafe { libc::mkfifo(c_path.as_ptr(), 0o644) }, 0);
let (sender, receiver) = std::sync::mpsc::channel();
let opened_root = root.clone();
let worker = std::thread::spawn(move || {
let open_error = Source::open(opened_root.join("pipe.dat")).map(|_| ());
let directory = Source::open(&opened_root).unwrap();
let buffer_error = directory
.buffer(&ArtifactPath::new("pipe.dat").unwrap())
.map(|_| ());
let sibling = directory
.buffer(&ArtifactPath::new("real.csv").unwrap())
.map(|buffer| buffer.bytes().to_vec());
sender.send((open_error, buffer_error, sibling)).unwrap();
});
let (open_error, buffer_error, sibling) = receiver
.recv_timeout(std::time::Duration::from_secs(10))
.expect("acquisition on a writerless pipe completes promptly");
worker.join().unwrap();
assert_eq!(
open_error.unwrap_err().category(),
crate::ErrorCategory::Request
);
assert_eq!(
buffer_error.unwrap_err().category(),
crate::ErrorCategory::Request
);
assert_eq!(sibling.unwrap(), b"real");
std::fs::remove_dir_all(root).unwrap();
}
#[test]
fn an_over_budget_referenced_file_is_refused_before_allocation() {
let root = test_root("budget");
std::fs::create_dir_all(&root).unwrap();
std::fs::write(root.join("master.dss"), b"master").unwrap();
let big = std::fs::File::create(root.join("big.dat")).unwrap();
big.set_len(MAX_REFERENCED_BYTES * 4).unwrap();
drop(big);
let source = Source::open(root.join("master.dss")).unwrap();
let primary = source.primary_buffer().unwrap();
let (error, allocated) =
measured_bytes(|| source.referenced_buffer(&primary, "big.dat").unwrap_err());
assert!(error.to_string().contains("acquisition budget"), "{error}");
assert!(
(allocated as u64) < MAX_REFERENCED_BYTES / 16,
"the refused acquisition allocated {allocated} bytes"
);
std::fs::remove_dir_all(root).unwrap();
}
#[cfg(unix)]
#[test]
fn racing_entry_listing_never_names_files_outside_the_root() {
use std::os::unix::fs::symlink;
let root = test_root("race-list");
std::fs::create_dir_all(root.join("sub")).unwrap();
std::fs::write(root.join("sub/inside.txt"), b"inside").unwrap();
let outside = test_root("race-list-outside");
std::fs::create_dir_all(&outside).unwrap();
std::fs::write(outside.join("outside-only.txt"), b"outside").unwrap();
let source = Source::open(&root).unwrap();
let stop = std::sync::atomic::AtomicBool::new(false);
std::thread::scope(|scope| {
let flipper = scope.spawn(|| {
while !stop.load(std::sync::atomic::Ordering::Relaxed) {
let _ = std::fs::remove_dir_all(root.join("sub"));
let _ = symlink(&outside, root.join("sub"));
let _ = std::fs::remove_file(root.join("sub"));
let _ = std::fs::create_dir(root.join("sub"));
let _ = std::fs::write(root.join("sub/inside.txt"), b"inside");
}
});
for _ in 0..50 {
if let Ok(names) = source.entry_names() {
assert!(
names
.iter()
.all(|name| !name.as_str().contains("outside-only")),
"{names:?}"
);
}
}
stop.store(true, std::sync::atomic::Ordering::Relaxed);
flipper.join().unwrap();
});
std::fs::remove_dir_all(&root).ok();
std::fs::remove_dir_all(&outside).ok();
}
#[test]
fn referenced_names_must_be_portable_relative_paths() {
let root = test_root("portable-names");
std::fs::create_dir_all(root.join("sub")).unwrap();
std::fs::write(root.join("master.dss"), b"master").unwrap();
std::fs::write(root.join("sub/feeder.dss"), b"feeder").unwrap();
let source = Source::open(root.join("master.dss")).unwrap();
let primary = source.primary_buffer().unwrap();
for name in ["..\\escape.dss", "\\escape.dss", "C:\\escape.dss", "C:x"] {
let error = source.referenced_buffer(&primary, name).unwrap_err();
assert_eq!(
error.category(),
crate::ErrorCategory::Request,
"{name}: {error}"
);
}
assert!(source.root_buffer("..\\master.dss").is_err());
assert!(source.root_buffer("\\master.dss").is_err());
let feeder = source
.referenced_buffer(&primary, "sub/feeder.dss")
.unwrap();
assert_eq!(feeder.bytes(), b"feeder");
let absolute = root.canonicalize().unwrap().join("sub").join("feeder.dss");
let again = source
.referenced_buffer(&primary, absolute.to_str().unwrap())
.unwrap();
assert_eq!(again.bytes().as_ptr(), feeder.bytes().as_ptr());
assert!(
source
.acquired_buffers()
.iter()
.all(|buffer| !buffer.name().contains("escape"))
);
std::fs::remove_dir_all(root).unwrap();
}
#[cfg(unix)]
#[test]
fn live_sources_hold_no_directory_descriptors_before_acquisition() {
fn open_descriptor_count() -> usize {
let table = if cfg!(target_os = "macos") {
"/dev/fd"
} else {
"/proc/self/fd"
};
std::fs::read_dir(table).unwrap().count()
}
let _guard = process_resource_guard();
let root = test_root("fd-count");
std::fs::create_dir_all(&root).unwrap();
std::fs::write(root.join("case.m"), b"case").unwrap();
std::fs::write(root.join("ref.csv"), b"ref").unwrap();
let before = open_descriptor_count();
let sources: Vec<Source> = (0..300)
.map(|_| Source::open(root.join("case.m")).unwrap())
.collect();
let held = open_descriptor_count();
assert!(
held <= before + 4,
"{} sources hold {} descriptors over the baseline {}",
sources.len(),
held - before,
before
);
for source in sources.iter().take(32) {
let primary = source.primary_buffer().unwrap();
let first = source.referenced_buffer(&primary, "ref.csv").unwrap();
let second = source.referenced_buffer(&primary, "ref.csv").unwrap();
assert_eq!(first.bytes().as_ptr(), second.bytes().as_ptr());
}
drop(sources);
std::fs::remove_dir_all(root).unwrap();
}
#[test]
fn entry_listing_returns_names_of_every_length_exactly() {
let root = test_root("name-lengths");
std::fs::create_dir_all(&root).unwrap();
let long = "n".repeat(200);
for name in ["a", "medium-name.csv", long.as_str()] {
std::fs::write(root.join(name), b"x").unwrap();
}
let source = Source::open(&root).unwrap();
let mut names: Vec<String> = source
.entry_names()
.unwrap()
.iter()
.map(|name| name.as_str().to_owned())
.collect();
names.sort();
let mut expected = vec!["a".to_owned(), "medium-name.csv".to_owned(), long];
expected.sort();
assert_eq!(names, expected);
std::fs::remove_dir_all(root).unwrap();
}
#[cfg(unix)]
#[test]
fn a_directory_nested_past_the_depth_bound_is_refused_promptly() {
use std::os::fd::{AsRawFd, FromRawFd};
let _guard = process_resource_guard();
let root = test_root("deep-chain");
std::fs::create_dir_all(&root).unwrap();
let name = std::ffi::CString::new("d").unwrap();
let mut level = std::fs::File::open(&root).unwrap();
for _ in 0..(MAX_REFERENCED_DEPTH + 40) {
unsafe {
assert_eq!(libc::mkdirat(level.as_raw_fd(), name.as_ptr(), 0o755), 0);
let fd = libc::openat(
level.as_raw_fd(),
name.as_ptr(),
libc::O_RDONLY | libc::O_CLOEXEC,
);
assert!(fd >= 0);
level = std::fs::File::from_raw_fd(fd);
}
}
drop(level);
let (sender, receiver) = std::sync::mpsc::channel();
let listed_root = root.clone();
let worker = std::thread::spawn(move || {
let source = Source::open(&listed_root).unwrap();
sender.send(source.entry_names().map(|_| ())).unwrap();
});
let outcome = receiver
.recv_timeout(std::time::Duration::from_secs(10))
.expect("the depth refusal returns promptly");
worker.join().unwrap();
let error = outcome.expect_err("a chain past the depth bound refuses");
assert!(error.to_string().contains("levels deep"), "{error}");
let mut fds = vec![std::fs::File::open(&root).unwrap()];
loop {
let last = fds.last().unwrap();
let fd = unsafe {
libc::openat(
last.as_raw_fd(),
name.as_ptr(),
libc::O_RDONLY | libc::O_CLOEXEC,
)
};
if fd < 0 {
break;
}
fds.push(unsafe { std::fs::File::from_raw_fd(fd) });
}
while fds.len() > 1 {
let parent = &fds[fds.len() - 2];
unsafe {
libc::unlinkat(parent.as_raw_fd(), name.as_ptr(), libc::AT_REMOVEDIR);
}
fds.pop();
}
drop(fds);
std::fs::remove_dir_all(&root).unwrap();
}
#[test]
fn a_directory_past_the_entry_budget_is_refused_with_bounded_memory() {
let root = test_root("entry-budget");
std::fs::create_dir_all(&root).unwrap();
let excess = MAX_REFERENCED_FILES * 4;
for index in 0..excess {
std::fs::write(root.join(format!("f{index:05}.csv")), b"").unwrap();
}
let source = Source::open(&root).unwrap();
let (error, allocated) = measured_bytes(|| source.entry_names().unwrap_err());
assert!(error.to_string().contains("entries"), "{error}");
assert!(
allocated < MAX_REFERENCED_FILES * 192,
"the refused listing allocated {allocated} bytes"
);
std::fs::remove_dir_all(root).unwrap();
}
#[cfg(unix)]
#[test]
fn listing_breadth_never_scales_open_descriptors() {
fn open_descriptor_count() -> usize {
let table = if cfg!(target_os = "macos") {
"/dev/fd"
} else {
"/proc/self/fd"
};
std::fs::read_dir(table).unwrap().count()
}
let _guard = process_resource_guard();
let root = test_root("breadth");
let breadth = 400usize;
for index in 0..breadth {
let sub = root.join(format!("s{index:03}"));
std::fs::create_dir_all(&sub).unwrap();
std::fs::write(sub.join("data.csv"), b"x").unwrap();
}
std::fs::create_dir_all(root.join("nested/a/b/c")).unwrap();
std::fs::write(root.join("nested/a/b/c/deep.csv"), b"x").unwrap();
let mut original: libc::rlimit = unsafe { std::mem::zeroed() };
assert_eq!(
unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &raw mut original) },
0
);
let lowered = libc::rlimit {
rlim_cur: 256,
rlim_max: original.rlim_max,
};
assert_eq!(
unsafe { libc::setrlimit(libc::RLIMIT_NOFILE, &raw const lowered) },
0
);
let baseline = open_descriptor_count();
let peak = std::sync::atomic::AtomicUsize::new(0);
let stop = std::sync::atomic::AtomicBool::new(false);
let names = std::thread::scope(|scope| {
let sampler = scope.spawn(|| {
while !stop.load(std::sync::atomic::Ordering::Relaxed) {
let count = open_descriptor_count();
peak.fetch_max(count, std::sync::atomic::Ordering::Relaxed);
}
});
let source = Source::open(&root).unwrap();
let names = source.entry_names().unwrap();
stop.store(true, std::sync::atomic::Ordering::Relaxed);
sampler.join().unwrap();
drop(source);
names
});
assert_eq!(
unsafe { libc::setrlimit(libc::RLIMIT_NOFILE, &raw const original) },
0
);
assert_eq!(names.len(), breadth + 1, "every file listed");
let sampled_peak = peak.load(std::sync::atomic::Ordering::Relaxed);
assert!(
sampled_peak <= baseline + MAX_REFERENCED_DEPTH + 16,
"the walk held {sampled_peak} descriptors over a baseline of {baseline}"
);
std::fs::remove_dir_all(root).unwrap();
}
#[test]
fn a_directory_listing_still_names_windows_reserved_spellings() {
let root = test_root("reserved-listing");
std::fs::create_dir_all(&root).unwrap();
std::fs::write(root.join("aux.dss"), b"content").unwrap();
let source = Source::open(&root).unwrap();
let names = source.entry_names().unwrap();
assert_eq!(names.len(), 1);
assert_eq!(names[0].as_str(), "aux.dss");
std::fs::remove_dir_all(root).unwrap();
}
#[test]
fn a_directory_listing_repeats_and_survives_acquisition() {
let root = test_root("repeat-listing");
std::fs::create_dir_all(&root).unwrap();
std::fs::write(root.join("network.csv"), b"name\nseq\n").unwrap();
std::fs::write(root.join("buses.csv"), b"name\nB1\n").unwrap();
let source = Source::open(&root).unwrap();
let first = source.entry_names().unwrap();
assert_eq!(first.len(), 2);
let name = ArtifactPath::new("network.csv").unwrap();
source.buffer(&name).unwrap();
let second = source.entry_names().unwrap();
assert_eq!(first, second);
std::fs::remove_dir_all(root).unwrap();
}
#[test]
fn concurrent_acquisition_retains_one_buffer_for_one_name() {
let root = test_root("concurrent");
std::fs::create_dir_all(&root).unwrap();
std::fs::write(root.join("shared.csv"), b"shared").unwrap();
let source = Source::open(&root).unwrap();
let name = ArtifactPath::new("shared.csv").unwrap();
let buffers: Vec<_> = std::thread::scope(|scope| {
(0..8)
.map(|_| {
let source = source.clone();
let name = name.clone();
scope.spawn(move || source.buffer(&name).unwrap())
})
.collect::<Vec<_>>()
.into_iter()
.map(|handle| handle.join().unwrap())
.collect()
});
let pointer = buffers[0].bytes().as_ptr();
assert!(
buffers
.iter()
.all(|buffer| buffer.bytes().as_ptr() == pointer)
);
assert_eq!(source.acquired_buffers().len(), 1);
std::fs::remove_dir_all(root).unwrap();
}
#[cfg(unix)]
#[test]
fn a_refused_listing_never_shortens_the_next_one() {
let root = test_root("refused-listing");
std::fs::create_dir_all(&root).unwrap();
for index in 0..(MAX_REFERENCED_FILES + 5) {
std::fs::write(root.join(format!("f{index}.txt")), b"x").unwrap();
}
let source = Source::open(&root).unwrap();
let first = source.entry_names().unwrap_err();
let second = source.entry_names().unwrap_err();
assert_eq!(
first.diagnostics().first().map(|d| d.code().to_owned()),
second.diagnostics().first().map(|d| d.code().to_owned()),
"the refusal repeats rather than shrinking into a partial listing"
);
drop(source);
let _ = std::fs::remove_dir_all(&root);
}
}