use std::io::Read;
use std::path::{Path, PathBuf};
use ignore::gitignore::{Gitignore, GitignoreBuilder};
use crate::error_display::format_error_list;
use crate::fs_traversal::{self, TraversalPolicy, UnsupportedEntry};
use crate::limits::{LimitError, Limits, ResourceKind};
use crate::pack::names_pack_path;
use crate::project_snapshot::{ProjectSnapshot, ProjectSnapshotAssembly, ProjectSnapshotError};
pub const IGNORE_FILE: &str = ".typkignore";
pub type FilesystemProjectResource = ResourceKind<0>;
#[allow(non_upper_case_globals)]
impl ResourceKind<0> {
pub const VisitedEntries: Self = Self::new(0);
pub const SelectedFiles: Self = Self::new(1);
pub const RootPolicyBytes: Self = Self::new(2);
pub const SelectedFileBytes: Self = Self::new(3);
pub const TotalSelectedBytes: Self = Self::new(4);
}
pub type FilesystemProjectLimitError = LimitError<FilesystemProjectResource>;
pub type FilesystemProjectLimits = Limits<FilesystemProjectResource>;
impl Limits<FilesystemProjectResource> {
#[track_caller]
pub fn new(
visited_entries: u64,
selected_files: u64,
root_policy_bytes: u64,
selected_file_bytes: u64,
total_selected_bytes: u64,
) -> Self {
Self::from_ceilings([
visited_entries,
selected_files,
root_policy_bytes,
selected_file_bytes,
total_selected_bytes,
0,
0,
])
.assert_probe_resources([
FilesystemProjectResource::VisitedEntries,
FilesystemProjectResource::SelectedFiles,
FilesystemProjectResource::RootPolicyBytes,
FilesystemProjectResource::SelectedFileBytes,
FilesystemProjectResource::TotalSelectedBytes,
])
}
pub const fn reference_v1() -> Self {
Self::from_ceilings([
1_000_000,
100_000,
1024 * 1024,
256 * 1024 * 1024,
2 * 1024 * 1024 * 1024,
0,
0,
])
}
pub const fn visited_entries(&self) -> u64 {
self.ceilings[0]
}
pub const fn selected_files(&self) -> u64 {
self.ceilings[1]
}
pub const fn root_policy_bytes(&self) -> u64 {
self.ceilings[2]
}
pub const fn selected_file_bytes(&self) -> u64 {
self.ceilings[3]
}
pub const fn total_selected_bytes(&self) -> u64 {
self.ceilings[4]
}
}
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
#[non_exhaustive]
pub enum FilesystemProjectEntryKind {
Socket,
Fifo,
BlockDevice,
CharacterDevice,
Unknown,
}
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
#[non_exhaustive]
pub enum FilesystemProjectOperation {
InspectRootPolicy,
ReadRootPolicy,
SurveyEntry,
InspectSelectedFile,
ReadSelectedFile,
}
impl std::fmt::Display for FilesystemProjectOperation {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(match self {
Self::InspectRootPolicy => "inspect root Project Ignore Policy",
Self::ReadRootPolicy => "read root Project Ignore Policy",
Self::SurveyEntry => "survey project entry",
Self::InspectSelectedFile => "inspect selected project file",
Self::ReadSelectedFile => "read selected project file",
})
}
}
#[derive(Debug, Clone, Eq, PartialEq, thiserror::Error)]
#[non_exhaustive]
pub enum FilesystemProjectIssue {
#[error("unsupported filesystem entry `{}`: aliases cannot become project files", path.display())]
Alias { path: PathBuf },
#[error("unsupported filesystem entry `{}` in the project", path.display())]
UnsupportedEntry {
path: PathBuf,
kind: FilesystemProjectEntryKind,
},
#[error("project path `{}` is not valid UTF-8", path.display())]
UnrepresentablePath { path: PathBuf },
}
impl FilesystemProjectIssue {
fn path(&self) -> &Path {
match self {
Self::Alias { path }
| Self::UnsupportedEntry { path, .. }
| Self::UnrepresentablePath { path } => path,
}
}
fn rank(&self) -> u8 {
match self {
Self::Alias { .. } => 0,
Self::UnsupportedEntry { .. } => 1,
Self::UnrepresentablePath { .. } => 2,
}
}
}
#[derive(Debug, Clone, Eq, PartialEq, thiserror::Error)]
#[error(
"filesystem project survey found {} issue(s){}",
.issues.len(),
format_error_list(.issues.as_slice())
)]
pub struct FilesystemProjectSurveyError {
issues: Vec<FilesystemProjectIssue>,
}
impl FilesystemProjectSurveyError {
pub fn issues(&self) -> &[FilesystemProjectIssue] {
&self.issues
}
}
#[derive(Debug, Clone, Eq, PartialEq, thiserror::Error)]
#[non_exhaustive]
pub enum FilesystemProjectPolicyError {
#[error("the policy file is not valid UTF-8")]
NotUtf8,
#[error("line {line}: {message}")]
InvalidRule { line: usize, message: String },
#[error("{0}")]
Invalid(String),
}
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum FilesystemProjectReadError {
#[error("failed to {operation} `{}`: {source}", path.display())]
Io {
operation: FilesystemProjectOperation,
path: PathBuf,
#[source]
source: std::io::Error,
},
#[error("invalid Project Ignore Policy at `{}`: {source}", path.display())]
InvalidPolicy {
path: PathBuf,
source: FilesystemProjectPolicyError,
},
#[error(transparent)]
Survey(FilesystemProjectSurveyError),
#[error("filesystem project resource limit at {path:?}: {source}")]
Limit {
path: PathBuf,
source: FilesystemProjectLimitError,
},
#[error("selected filesystem entries do not form a Project Snapshot: {0}")]
Snapshot(#[source] ProjectSnapshotError),
}
impl FilesystemProjectReadError {
fn io(
operation: FilesystemProjectOperation,
path: impl Into<PathBuf>,
source: std::io::Error,
) -> Self {
Self::Io {
operation,
path: path.into(),
source,
}
}
fn limit(path: impl Into<PathBuf>, source: FilesystemProjectLimitError) -> Self {
Self::Limit {
path: path.into(),
source,
}
}
}
pub fn read_filesystem_project(
root: impl AsRef<Path>,
entrypoint: impl Into<String>,
limits: FilesystemProjectLimits,
) -> Result<ProjectSnapshot, FilesystemProjectReadError> {
let root = root.as_ref();
let policy_path = root.join(IGNORE_FILE);
let policy_file = read_policy(&policy_path, limits)?;
let policy = &policy_file.policy;
let mut visited_entries = 0u64;
let mut selected_files = u64::from(policy_file.bytes.is_some());
check_limit(
FilesystemProjectResource::SelectedFiles,
limits.selected_files(),
selected_files,
)
.map_err(|source| FilesystemProjectReadError::limit(&policy_path, source))?;
let mut declared_total = policy_file
.bytes
.as_ref()
.map_or(0, |bytes| bytes.len() as u64);
let mut selected = Vec::new();
let mut issues = Vec::new();
let mut deferred_limit = None;
let mut walk = walkdir::WalkDir::new(root).follow_links(false).into_iter();
while let Some(entry) = walk.next() {
let entry = entry.map_err(|error| {
let path = error.path().unwrap_or(root).to_owned();
let source = error
.into_io_error()
.unwrap_or_else(|| std::io::Error::other("filesystem traversal failed"));
FilesystemProjectReadError::io(FilesystemProjectOperation::SurveyEntry, path, source)
})?;
if entry.depth() == 0 {
continue;
}
visited_entries = checked_add(
visited_entries,
1,
FilesystemProjectResource::VisitedEntries,
)
.map_err(|source| FilesystemProjectReadError::limit(entry.path(), source))?;
check_limit(
FilesystemProjectResource::VisitedEntries,
limits.visited_entries(),
visited_entries,
)
.map_err(|source| FilesystemProjectReadError::limit(entry.path(), source))?;
let relative = entry
.path()
.strip_prefix(root)
.expect("walk remains beneath root");
let Some(path) = slash_path(relative) else {
issues.push(FilesystemProjectIssue::UnrepresentablePath {
path: entry.path().to_owned(),
});
if entry.file_type().is_dir() {
walk.skip_current_dir();
}
continue;
};
if path == IGNORE_FILE {
if entry.file_type().is_dir() {
walk.skip_current_dir();
}
continue;
}
let file_type = entry.file_type();
if file_type.is_dir() {
if policy.excludes_directory(&path) {
walk.skip_current_dir();
}
continue;
}
if policy.excludes_file(&path) {
continue;
}
if file_type.is_symlink() {
issues.push(FilesystemProjectIssue::Alias {
path: entry.path().to_owned(),
});
continue;
}
if !file_type.is_file() {
issues.push(FilesystemProjectIssue::UnsupportedEntry {
path: entry.path().to_owned(),
kind: UnsupportedEntry::of(&file_type).into(),
});
continue;
}
selected_files =
checked_add(selected_files, 1, FilesystemProjectResource::SelectedFiles)
.map_err(|source| FilesystemProjectReadError::limit(entry.path(), source))?;
if let Err(source) = check_limit(
FilesystemProjectResource::SelectedFiles,
limits.selected_files(),
selected_files,
) {
deferred_limit
.get_or_insert_with(|| FilesystemProjectReadError::limit(entry.path(), source));
continue;
}
let metadata = entry.metadata().map_err(|error| {
let path = error.path().unwrap_or(entry.path()).to_owned();
let source = error.into_io_error().unwrap_or_else(|| {
std::io::Error::other("failed to inspect selected project file")
});
FilesystemProjectReadError::io(
FilesystemProjectOperation::InspectSelectedFile,
path,
source,
)
})?;
let declared = metadata.len();
if let Err(source) = check_limit(
FilesystemProjectResource::SelectedFileBytes,
limits.selected_file_bytes(),
declared,
) {
deferred_limit
.get_or_insert_with(|| FilesystemProjectReadError::limit(entry.path(), source));
continue;
}
declared_total = checked_add(
declared_total,
declared,
FilesystemProjectResource::TotalSelectedBytes,
)
.map_err(|source| FilesystemProjectReadError::limit(entry.path(), source))?;
if let Err(source) = check_limit(
FilesystemProjectResource::TotalSelectedBytes,
limits.total_selected_bytes(),
declared_total,
) {
deferred_limit
.get_or_insert_with(|| FilesystemProjectReadError::limit(entry.path(), source));
continue;
}
selected.push((path, entry.path().to_owned()));
}
if !issues.is_empty() {
issues.sort_by(|left, right| {
left.path()
.cmp(right.path())
.then_with(|| left.rank().cmp(&right.rank()))
});
return Err(FilesystemProjectReadError::Survey(
FilesystemProjectSurveyError { issues },
));
}
if let Some(error) = deferred_limit {
return Err(error);
}
selected.sort_by(|(left, _), (right, _)| left.cmp(right));
let mut entries = Vec::with_capacity(selected.len() + usize::from(policy_file.bytes.is_some()));
let mut actual_total = 0u64;
if let Some(bytes) = policy_file.bytes {
actual_total = bytes.len() as u64;
entries.push((IGNORE_FILE.to_owned(), bytes));
}
for (path, source) in selected {
let remaining = limits.total_selected_bytes() - actual_total;
let bytes = read_bounded(
root,
&source,
&[
(
FilesystemProjectResource::SelectedFileBytes,
limits.selected_file_bytes(),
limits.selected_file_bytes(),
0,
),
(
FilesystemProjectResource::TotalSelectedBytes,
remaining,
limits.total_selected_bytes(),
actual_total,
),
],
FilesystemProjectOperation::ReadSelectedFile,
)?;
actual_total = checked_add(
actual_total,
bytes.len() as u64,
FilesystemProjectResource::TotalSelectedBytes,
)
.map_err(|source_error| FilesystemProjectReadError::limit(&source, source_error))?;
entries.push((path, bytes));
}
ProjectSnapshotAssembly::new(entrypoint)
.assemble(entries)
.map_err(FilesystemProjectReadError::Snapshot)
}
struct ReadPolicy {
policy: ProjectIgnorePolicy,
bytes: Option<Vec<u8>>,
}
fn read_policy(
path: &Path,
limits: FilesystemProjectLimits,
) -> Result<ReadPolicy, FilesystemProjectReadError> {
let metadata = match std::fs::symlink_metadata(path) {
Ok(metadata) => metadata,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
return Ok(ReadPolicy {
policy: ProjectIgnorePolicy::built_in(),
bytes: None,
});
}
Err(error) => {
return Err(FilesystemProjectReadError::io(
FilesystemProjectOperation::InspectRootPolicy,
path,
error,
));
}
};
let file_type = metadata.file_type();
if file_type.is_symlink() {
return Err(FilesystemProjectReadError::Survey(
FilesystemProjectSurveyError {
issues: vec![FilesystemProjectIssue::Alias {
path: path.to_owned(),
}],
},
));
}
if !file_type.is_file() {
return Err(FilesystemProjectReadError::Survey(
FilesystemProjectSurveyError {
issues: vec![FilesystemProjectIssue::UnsupportedEntry {
path: path.to_owned(),
kind: UnsupportedEntry::of(&file_type).into(),
}],
},
));
}
check_limit(
FilesystemProjectResource::VisitedEntries,
limits.visited_entries(),
1,
)
.and_then(|_| {
check_limit(
FilesystemProjectResource::SelectedFiles,
limits.selected_files(),
1,
)
})
.map_err(|source| FilesystemProjectReadError::limit(path, source))?;
check_limit(
FilesystemProjectResource::RootPolicyBytes,
limits.root_policy_bytes(),
metadata.len(),
)
.and_then(|_| {
check_limit(
FilesystemProjectResource::SelectedFileBytes,
limits.selected_file_bytes(),
metadata.len(),
)
})
.and_then(|_| {
check_limit(
FilesystemProjectResource::TotalSelectedBytes,
limits.total_selected_bytes(),
metadata.len(),
)
})
.map_err(|source| FilesystemProjectReadError::limit(path, source))?;
let bytes = read_bounded(
path.parent().expect("the policy path is beneath a root"),
path,
&[
(
FilesystemProjectResource::RootPolicyBytes,
limits.root_policy_bytes(),
limits.root_policy_bytes(),
0,
),
(
FilesystemProjectResource::SelectedFileBytes,
limits.selected_file_bytes(),
limits.selected_file_bytes(),
0,
),
(
FilesystemProjectResource::TotalSelectedBytes,
limits.total_selected_bytes(),
limits.total_selected_bytes(),
0,
),
],
FilesystemProjectOperation::ReadRootPolicy,
)?;
let policy = ProjectIgnorePolicy::from_bytes(&bytes).map_err(|source| {
FilesystemProjectReadError::InvalidPolicy {
path: path.to_owned(),
source,
}
})?;
Ok(ReadPolicy {
policy,
bytes: Some(bytes),
})
}
fn read_bounded(
root: &Path,
path: &Path,
ceilings: &[(FilesystemProjectResource, u64, u64, u64)],
operation: FilesystemProjectOperation,
) -> Result<Vec<u8>, FilesystemProjectReadError> {
let probe_ceiling = ceilings
.iter()
.map(|(_, allowance, _, _)| *allowance)
.min()
.expect("a bounded read has at least one ceiling");
let mut file =
fs_traversal::open_without_following(&ProjectTraversal { operation }, root, path)?;
let mut bytes = Vec::new();
file.by_ref()
.take(probe_ceiling + 1)
.read_to_end(&mut bytes)
.map_err(|error| FilesystemProjectReadError::io(operation, path, error))?;
let observed = u64::try_from(bytes.len()).map_err(|_| {
FilesystemProjectReadError::limit(
path,
FilesystemProjectLimitError::AccountingOverflow {
resource: ceilings[0].0,
},
)
})?;
for (resource, _, ceiling, base) in ceilings {
let cumulative = checked_add(*base, observed, *resource)
.map_err(|source| FilesystemProjectReadError::limit(path, source))?;
check_limit(*resource, *ceiling, cumulative)
.map_err(|source| FilesystemProjectReadError::limit(path, source))?;
}
Ok(bytes)
}
struct ProjectTraversal {
operation: FilesystemProjectOperation,
}
impl TraversalPolicy for ProjectTraversal {
type Error = FilesystemProjectReadError;
const ROOT_INVARIANT: &'static str = "a selected path remains beneath its project root";
fn io(&self, path: &Path, source: std::io::Error) -> Self::Error {
FilesystemProjectReadError::io(self.operation, path, source)
}
fn alias(&self, path: &Path) -> Self::Error {
alias_error(path)
}
fn unsupported_entry(&self, path: &Path, entry: UnsupportedEntry) -> Self::Error {
FilesystemProjectReadError::Survey(FilesystemProjectSurveyError {
issues: vec![FilesystemProjectIssue::UnsupportedEntry {
path: path.to_owned(),
kind: entry.into(),
}],
})
}
}
impl From<UnsupportedEntry> for FilesystemProjectEntryKind {
fn from(entry: UnsupportedEntry) -> Self {
match entry {
UnsupportedEntry::Socket => Self::Socket,
UnsupportedEntry::Fifo => Self::Fifo,
UnsupportedEntry::BlockDevice => Self::BlockDevice,
UnsupportedEntry::CharacterDevice => Self::CharacterDevice,
UnsupportedEntry::Unknown => Self::Unknown,
}
}
}
fn alias_error(path: &Path) -> FilesystemProjectReadError {
FilesystemProjectReadError::Survey(FilesystemProjectSurveyError {
issues: vec![FilesystemProjectIssue::Alias {
path: path.to_owned(),
}],
})
}
fn slash_path(path: &Path) -> Option<String> {
path.components()
.map(|component| component.as_os_str().to_str())
.collect::<Option<Vec<_>>>()
.map(|components| components.join("/"))
}
fn checked_add(
total: u64,
value: u64,
resource: FilesystemProjectResource,
) -> Result<u64, FilesystemProjectLimitError> {
total
.checked_add(value)
.ok_or(FilesystemProjectLimitError::AccountingOverflow { resource })
}
fn check_limit(
resource: FilesystemProjectResource,
ceiling: u64,
observed: u64,
) -> Result<(), FilesystemProjectLimitError> {
if observed > ceiling {
return Err(FilesystemProjectLimitError::exceeded(resource, ceiling));
}
Ok(())
}
struct ProjectIgnorePolicy {
rules: Gitignore,
}
impl ProjectIgnorePolicy {
fn built_in() -> Self {
Self {
rules: Gitignore::empty(),
}
}
fn from_bytes(bytes: &[u8]) -> Result<Self, FilesystemProjectPolicyError> {
let contents =
std::str::from_utf8(bytes).map_err(|_| FilesystemProjectPolicyError::NotUtf8)?;
let mut builder = GitignoreBuilder::new(".");
for (index, line) in contents.lines().enumerate() {
let line = if index == 0 {
line.trim_start_matches('\u{feff}')
} else {
line
};
builder.add_line(None, line).map_err(|error| {
FilesystemProjectPolicyError::InvalidRule {
line: index + 1,
message: error.to_string(),
}
})?;
}
let rules = builder
.build()
.map_err(|error| FilesystemProjectPolicyError::Invalid(error.to_string()))?;
Ok(Self { rules })
}
fn excludes_file(&self, path: &str) -> bool {
self.excludes(path, false)
}
fn excludes_directory(&self, path: &str) -> bool {
self.excludes(path, true)
}
fn excludes(&self, path: &str, is_directory: bool) -> bool {
if path == IGNORE_FILE {
return false;
}
if names_pack_path(path) {
return true;
}
let mut ancestor_end = 0;
while let Some(offset) = path[ancestor_end..].find('/') {
ancestor_end += offset;
if self.rules.matched(&path[..ancestor_end], true).is_ignore() {
return true;
}
ancestor_end += 1;
}
self.rules.matched(path, is_directory).is_ignore()
}
}
#[cfg(all(test, unix))]
mod tests {
use std::os::unix::fs::symlink;
use super::*;
#[test]
fn bounded_reads_do_not_follow_an_alias_created_after_survey() {
let directory = tempfile::tempdir().unwrap();
let outside = directory.path().join("outside");
let selected = directory.path().join("selected");
std::fs::write(&outside, b"outside").unwrap();
symlink(&outside, &selected).unwrap();
let error = read_bounded(
directory.path(),
&selected,
&[(FilesystemProjectResource::SelectedFileBytes, 16, 16, 0)],
FilesystemProjectOperation::ReadSelectedFile,
)
.unwrap_err();
assert!(matches!(
error,
FilesystemProjectReadError::Survey(ref survey)
if matches!(survey.issues(), [FilesystemProjectIssue::Alias { path }] if path == &selected)
));
}
#[test]
fn bounded_reads_do_not_follow_an_ancestor_alias_created_after_survey() {
let directory = tempfile::tempdir().unwrap();
let root = directory.path().join("project");
let ancestor = root.join("nested");
let selected = ancestor.join("selected");
let outside = directory.path().join("outside");
std::fs::create_dir_all(&ancestor).unwrap();
std::fs::create_dir_all(&outside).unwrap();
std::fs::write(&selected, b"surveyed").unwrap();
std::fs::write(outside.join("selected"), b"outside").unwrap();
std::fs::remove_dir_all(&ancestor).unwrap();
symlink(&outside, &ancestor).unwrap();
let error = read_bounded(
&root,
&selected,
&[(FilesystemProjectResource::SelectedFileBytes, 16, 16, 0)],
FilesystemProjectOperation::ReadSelectedFile,
)
.unwrap_err();
assert!(matches!(
error,
FilesystemProjectReadError::Survey(ref survey)
if matches!(survey.issues(), [FilesystemProjectIssue::Alias { path }] if path == &ancestor)
));
}
#[test]
fn bounded_reads_do_not_block_on_a_fifo_created_after_survey() {
use std::ffi::CString;
use std::os::unix::ffi::OsStrExt;
let directory = tempfile::tempdir().unwrap();
let selected = directory.path().join("selected");
std::fs::write(&selected, b"surveyed").unwrap();
std::fs::remove_file(&selected).unwrap();
let path = CString::new(selected.as_os_str().as_bytes()).unwrap();
assert_eq!(unsafe { libc::mkfifo(path.as_ptr(), 0o600) }, 0);
let error = read_bounded(
directory.path(),
&selected,
&[(FilesystemProjectResource::SelectedFileBytes, 16, 16, 0)],
FilesystemProjectOperation::ReadSelectedFile,
)
.unwrap_err();
assert!(matches!(
error,
FilesystemProjectReadError::Survey(ref survey)
if matches!(survey.issues(), [FilesystemProjectIssue::UnsupportedEntry {
path,
kind: FilesystemProjectEntryKind::Fifo,
}] if path == &selected)
));
}
}