use crate::filter::{FilterResult, FilterSettings};
use crate::progress::Progress;
use anyhow::Context;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EntryKind {
File,
Dir,
Symlink,
Special,
}
impl EntryKind {
#[must_use]
pub fn from_metadata(metadata: &std::fs::Metadata) -> Self {
if metadata.is_dir() {
Self::Dir
} else if metadata.is_symlink() {
Self::Symlink
} else if metadata.is_file() {
Self::File
} else {
Self::Special
}
}
#[must_use]
pub fn label(self) -> &'static str {
match self {
Self::Dir => "dir",
Self::Symlink => "symlink",
Self::File | Self::Special => "file",
}
}
#[must_use]
pub fn label_long(self) -> &'static str {
match self {
Self::Dir => "directory",
Self::Symlink => "symlink",
Self::File | Self::Special => "file",
}
}
pub fn inc_skipped(self, prog: &Progress) {
match self {
Self::Dir => prog.directories_skipped.inc(),
Self::Symlink => prog.symlinks_skipped.inc(),
Self::File | Self::Special => prog.files_skipped.inc(),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PermitKind {
OpenFile,
PendingMeta,
None,
}
pub enum LeafPermit {
OpenFile(throttle::OpenFileGuard),
PendingMeta(throttle::PendingMetaGuard),
}
pub async fn preacquire_leaf_permit(
kind: PermitKind,
hint: Option<EntryKind>,
want: impl Fn(Option<EntryKind>) -> bool,
) -> Option<LeafPermit> {
if kind == PermitKind::None || !want(hint) {
return None;
}
match kind {
PermitKind::OpenFile => Some(LeafPermit::OpenFile(throttle::open_file_permit().await)),
PermitKind::PendingMeta => Some(LeafPermit::PendingMeta(
throttle::pending_meta_permit().await,
)),
PermitKind::None => None,
}
}
pub(crate) fn throttle_side(side: congestion::Side) -> throttle::Side {
match side {
congestion::Side::Source => throttle::Side::Source,
congestion::Side::Destination => throttle::Side::Destination,
}
}
pub(crate) fn throttle_op(op: congestion::MetadataOp) -> throttle::MetadataOp {
match op {
congestion::MetadataOp::Stat => throttle::MetadataOp::Stat,
congestion::MetadataOp::ReadLink => throttle::MetadataOp::ReadLink,
congestion::MetadataOp::MkDir => throttle::MetadataOp::MkDir,
congestion::MetadataOp::RmDir => throttle::MetadataOp::RmDir,
congestion::MetadataOp::Unlink => throttle::MetadataOp::Unlink,
congestion::MetadataOp::HardLink => throttle::MetadataOp::HardLink,
congestion::MetadataOp::Symlink => throttle::MetadataOp::Symlink,
congestion::MetadataOp::Chmod => throttle::MetadataOp::Chmod,
congestion::MetadataOp::OpenCreate => throttle::MetadataOp::OpenCreate,
}
}
pub(crate) fn meta_resource(
side: congestion::Side,
op: congestion::MetadataOp,
) -> throttle::Resource {
throttle::Resource::meta(throttle_side(side), throttle_op(op))
}
pub async fn next_entry_probed<F>(
entries: &mut tokio::fs::ReadDir,
_side: congestion::Side,
context: F,
) -> anyhow::Result<Option<(tokio::fs::DirEntry, Option<std::fs::FileType>)>>
where
F: FnOnce() -> String,
{
throttle::get_ops_token().await;
let maybe_entry = entries.next_entry().await.with_context(context)?;
let Some(entry) = maybe_entry else {
return Ok(None);
};
let entry_file_type = entry.file_type().await.ok();
Ok(Some((entry, entry_file_type)))
}
pub async fn run_metadata_probed<F, T, E>(
side: congestion::Side,
op_kind: congestion::MetadataOp,
fut: F,
) -> Result<T, E>
where
F: std::future::Future<Output = Result<T, E>>,
{
throttle::get_ops_token().await;
run_metadata_probed_no_rate(side, op_kind, fut).await
}
pub async fn run_metadata_probed_no_rate<F, T, E>(
side: congestion::Side,
op_kind: congestion::MetadataOp,
fut: F,
) -> Result<T, E>
where
F: std::future::Future<Output = Result<T, E>>,
{
let ops_permit = throttle::ops_in_flight_permit(meta_resource(side, op_kind)).await;
let probe = congestion::Probe::start_metadata(side, op_kind);
let result = fut.await;
match &result {
Ok(_) => probe.complete_ok(0),
Err(_) => probe.discard(),
}
drop(ops_permit);
result
}
pub async fn filter_is_dir(
filter: Option<&FilterSettings>,
dir: &crate::safedir::Dir,
name: &std::ffi::OsStr,
hint: Option<EntryKind>,
force_authoritative: bool,
) -> bool {
match hint {
Some(kind) => kind == EntryKind::Dir,
None if filter.is_some() || force_authoritative => match dir.child(name).await {
Ok(handle) => handle.kind() == EntryKind::Dir,
Err(_) => false,
},
None => false,
}
}
#[must_use]
pub fn should_skip_entry(
filter: &Option<FilterSettings>,
relative_path: &std::path::Path,
is_dir: bool,
) -> Option<FilterResult> {
should_skip_entry_ref(filter.as_ref(), relative_path, is_dir)
}
#[must_use]
pub fn should_skip_entry_ref(
filter: Option<&FilterSettings>,
relative_path: &std::path::Path,
is_dir: bool,
) -> Option<FilterResult> {
if let Some(f) = filter {
let result = f.should_include(relative_path, is_dir);
match result {
FilterResult::Included => None,
_ => Some(result),
}
} else {
None
}
}
#[must_use]
pub fn relative_to_root<'a>(
entry: &'a std::path::Path,
root: &std::path::Path,
) -> &'a std::path::Path {
entry.strip_prefix(root).unwrap_or(entry)
}
#[must_use]
pub fn without_trailing_separators(path: &std::path::Path) -> std::path::PathBuf {
use std::os::unix::ffi::OsStrExt;
let bytes = path.as_os_str().as_bytes();
let mut end = bytes.len();
while end > 1 && bytes[end - 1] == b'/' {
end -= 1;
}
std::path::PathBuf::from(std::ffi::OsStr::from_bytes(&bytes[..end]))
}
pub struct RootOperand {
pub parent: std::path::PathBuf,
pub name: std::ffi::OsString,
pub display: std::path::PathBuf,
}
pub async fn split_root_operand(path: &std::path::Path) -> anyhow::Result<RootOperand> {
let stripped = without_trailing_separators(path);
if let Some(name) = stripped.file_name() {
let parent = match stripped.parent() {
Some(p) if !p.as_os_str().is_empty() => p.to_path_buf(),
_ => std::path::PathBuf::from("."),
};
let name = name.to_owned();
return Ok(RootOperand {
parent,
name,
display: stripped,
});
}
let canonical = tokio::fs::canonicalize(&stripped)
.await
.with_context(|| format!("cannot resolve operand {stripped:?}"))?;
let name = canonical.file_name().map(std::ffi::OsStr::to_owned);
let parent = canonical.parent().map(std::path::Path::to_path_buf);
let (Some(parent), Some(name)) = (parent, name) else {
anyhow::bail!("cannot operate on the filesystem root {canonical:?}");
};
Ok(RootOperand {
parent,
name,
display: canonical,
})
}
pub fn root_operand_basename(path: &std::path::Path) -> anyhow::Result<std::ffi::OsString> {
let stripped = without_trailing_separators(path);
if let Some(name) = stripped.file_name() {
return Ok(name.to_owned());
}
let canonical = std::fs::canonicalize(&stripped)
.with_context(|| format!("cannot resolve operand {stripped:?}"))?;
canonical
.file_name()
.map(std::ffi::OsStr::to_owned)
.ok_or_else(|| anyhow::anyhow!("cannot operate on the filesystem root {canonical:?}"))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::safedir::Dir;
use crate::testutils;
use std::ffi::OsStr;
fn include_filter(pattern: &str) -> Option<FilterSettings> {
let mut f = FilterSettings::new();
f.add_include(pattern).unwrap();
Some(f)
}
#[tokio::test]
async fn filter_is_dir_authoritatively_classifies_dt_unknown_directory() -> anyhow::Result<()> {
let tmp = testutils::setup_test_dir().await?;
let dir = Dir::open_root_dir(&tmp.join("foo"), false, congestion::Side::Source).await?;
let filter = include_filter("/bar/**");
assert!(
filter_is_dir(filter.as_ref(), &dir, OsStr::new("bar"), None, false).await,
"a real directory reported as DT_UNKNOWN must classify as a directory for the filter"
);
assert!(
!filter_is_dir(filter.as_ref(), &dir, OsStr::new("0.txt"), None, false).await,
"a real file reported as DT_UNKNOWN must classify as a non-directory"
);
Ok(())
}
#[tokio::test]
async fn filter_is_dir_uses_hint_when_available_and_skips_when_no_filter() -> anyhow::Result<()>
{
let tmp = testutils::setup_test_dir().await?;
let dir = Dir::open_root_dir(&tmp.join("foo"), false, congestion::Side::Source).await?;
let filter = include_filter("/bar/**");
assert!(
filter_is_dir(
filter.as_ref(),
&dir,
OsStr::new("bar"),
Some(EntryKind::Dir),
false
)
.await
);
assert!(
!filter_is_dir(
filter.as_ref(),
&dir,
OsStr::new("0.txt"),
Some(EntryKind::File),
false
)
.await
);
assert!(!filter_is_dir(None, &dir, OsStr::new("does_not_exist"), None, false).await);
Ok(())
}
#[tokio::test]
async fn filter_is_dir_forces_authoritative_classification_when_requested() -> anyhow::Result<()>
{
let tmp = testutils::setup_test_dir().await?;
let dir = Dir::open_root_dir(&tmp.join("foo"), false, congestion::Side::Source).await?;
assert!(
filter_is_dir(None, &dir, OsStr::new("bar"), None, true).await,
"force_authoritative must fstat a DT_UNKNOWN directory even with no filter"
);
assert!(
!filter_is_dir(None, &dir, OsStr::new("0.txt"), None, true).await,
"force_authoritative must fstat a DT_UNKNOWN file even with no filter"
);
Ok(())
}
#[tokio::test]
async fn preacquire_leaf_permit_respects_kind_and_want() {
throttle::set_max_open_files(4);
assert!(
preacquire_leaf_permit(PermitKind::None, Some(EntryKind::File), |_| true)
.await
.is_none()
);
let permit = preacquire_leaf_permit(PermitKind::OpenFile, Some(EntryKind::File), |h| {
h == Some(EntryKind::File)
})
.await;
assert!(matches!(permit, Some(LeafPermit::OpenFile(_))));
drop(permit);
assert!(
preacquire_leaf_permit(PermitKind::OpenFile, Some(EntryKind::Dir), |_| false)
.await
.is_none()
);
}
#[tokio::test]
async fn split_root_operand_handles_dot_and_normal_operands() -> anyhow::Result<()> {
use std::ffi::OsStr;
use std::path::Path;
let op = split_root_operand(Path::new("a/b")).await?;
assert_eq!(op.parent, Path::new("a"));
assert_eq!(op.name, OsStr::new("b"));
assert_eq!(op.display, Path::new("a/b"));
let op = split_root_operand(Path::new("foo")).await?;
assert_eq!(op.parent, Path::new("."));
assert_eq!(op.name, OsStr::new("foo"));
let op = split_root_operand(Path::new("foo/")).await?;
assert_eq!(op.name, OsStr::new("foo"));
let op = split_root_operand(Path::new("dir/.")).await?;
assert_eq!(op.parent, Path::new("."));
assert_eq!(op.name, OsStr::new("dir"));
let op = split_root_operand(Path::new("a/b/.")).await?;
assert_eq!(op.parent, Path::new("a"));
assert_eq!(op.name, OsStr::new("b"));
let cwd = tokio::fs::canonicalize(".").await?;
let op = split_root_operand(Path::new(".")).await?;
assert_eq!(op.parent, cwd.parent().unwrap());
assert_eq!(op.name, cwd.file_name().unwrap());
assert_eq!(op.display, cwd);
let tmp = testutils::create_temp_dir().await?;
let sub = tmp.join("sub");
tokio::fs::create_dir(&sub).await?;
let canonical_tmp = tokio::fs::canonicalize(&tmp).await?;
let op = split_root_operand(&sub.join("..")).await?;
assert_eq!(op.parent, canonical_tmp.parent().unwrap());
assert_eq!(op.name, canonical_tmp.file_name().unwrap());
assert_eq!(op.display, canonical_tmp);
assert!(split_root_operand(Path::new("/")).await.is_err());
Ok(())
}
#[tokio::test]
async fn root_operand_basename_matches_split_root_operand() -> anyhow::Result<()> {
use std::path::Path;
for p in ["a/b", "foo", "foo/", "dir/.", "a/b/."] {
assert_eq!(
root_operand_basename(Path::new(p))?,
split_root_operand(Path::new(p)).await?.name,
"operand {p:?}"
);
}
let tmp = testutils::create_temp_dir().await?;
let sub = tmp.join("sub");
tokio::fs::create_dir(&sub).await?;
let dot_operands = [
std::path::PathBuf::from("."),
std::path::PathBuf::from(".."),
sub.join(".."),
sub.join("../.."),
];
for p in &dot_operands {
assert_eq!(
root_operand_basename(p)?,
split_root_operand(p).await?.name,
"operand {p:?}"
);
}
assert!(root_operand_basename(Path::new("/")).is_err());
assert!(split_root_operand(Path::new("/")).await.is_err());
Ok(())
}
}