use std::fs::{self, File};
use std::io::{self, Read};
use std::path::Path;
use crate::error::{IdentifyError, Result};
use crate::filename::{tags_from_filename, tags_from_interpreter};
use crate::meta::{ExecPolicy, FileKind, Meta};
use crate::shebang::{Source, parse_shebang_buf};
use crate::tags::{Tag, TagSet};
use crate::text::{TEXT_PROBE_LEN, is_text, is_text_bytes, read_full};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Classification {
Done(TagSet),
NeedsContent(Pending),
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Pending {
tags: TagSet,
shebang: bool,
}
impl Pending {
#[must_use]
pub const fn tags(&self) -> TagSet {
self.tags
}
#[must_use]
pub const fn needs_shebang(&self) -> bool {
self.shebang
}
#[must_use]
pub const fn needs_encoding(&self) -> bool {
!self.tags.has_encoding()
}
pub fn finish<R: Read>(self, mut reader: R) -> io::Result<TagSet> {
let mut head = [0u8; TEXT_PROBE_LEN];
let n = read_full(&mut reader, &mut head)?;
self.finish_with_head(head.get(..n).unwrap_or_default(), reader)
}
#[must_use]
pub fn finish_bytes(self, content: &[u8]) -> TagSet {
let head = content.get(..TEXT_PROBE_LEN).unwrap_or(content);
let rest = content.get(head.len()..).unwrap_or_default();
self.finish_with_head(head, rest).unwrap_or(self.tags)
}
fn finish_with_head<R: Read>(self, head: &[u8], rest: R) -> io::Result<TagSet> {
let mut tags = self.tags;
if self.shebang {
let cmd = parse_shebang_buf(&mut Source::new(head, rest))?;
if let Some(interpreter) = cmd.first() {
tags = tags.union(tags_from_interpreter(interpreter));
}
}
if !tags.has_encoding() {
tags.insert(if is_text_bytes(head) {
Tag::TEXT
} else {
Tag::BINARY
});
}
Ok(tags)
}
}
#[must_use]
pub fn classify(meta: Meta, path: impl AsRef<[u8]>) -> Classification {
let kind = meta.kind();
let tags = match kind {
FileKind::Directory => return Classification::Done(TagSet::from_tag(Tag::DIRECTORY)),
FileKind::Symlink => return Classification::Done(TagSet::from_tag(Tag::SYMLINK)),
FileKind::Socket => return Classification::Done(TagSet::from_tag(Tag::SOCKET)),
FileKind::Regular | FileKind::Other => {
TagSet::from_tag(Tag::FILE).with(if meta.is_executable() {
Tag::EXECUTABLE
} else {
Tag::NON_EXECUTABLE
})
}
};
let name_tags = tags_from_filename(path);
let shebang = name_tags.is_empty() && meta.is_executable();
let mut tags = tags.union(name_tags);
if kind == FileKind::Other {
if !tags.has_encoding() {
tags.insert(Tag::BINARY);
}
return Classification::Done(tags);
}
if !shebang && tags.has_encoding() {
Classification::Done(tags)
} else {
Classification::NeedsContent(Pending { tags, shebang })
}
}
fn path_bytes(path: &Path) -> &[u8] {
path.as_os_str().as_encoded_bytes()
}
fn open_for_read(path: &Path) -> io::Result<File> {
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
let nonblock = i32::try_from(rustix::fs::OFlags::NONBLOCK.bits()).unwrap_or(0);
fs::OpenOptions::new()
.read(true)
.custom_flags(nonblock)
.open(path)
}
#[cfg(not(unix))]
{
File::open(path)
}
}
fn finish_from_file(path: &Path, pending: Pending, len_hint: Option<u64>) -> Result<TagSet> {
let file = open_for_read(path).map_err(|e| {
if e.kind() == io::ErrorKind::NotFound {
IdentifyError::not_found(path, e)
} else {
IdentifyError::from(e)
}
})?;
Ok(finish_sized(pending, file, len_hint)?)
}
fn finish_sized<R: Read>(
pending: Pending,
mut file: R,
len_hint: Option<u64>,
) -> io::Result<TagSet> {
let want = match len_hint {
Some(len) if len > 0 => {
usize::try_from(len).map_or(TEXT_PROBE_LEN, |l| l.min(TEXT_PROBE_LEN))
}
_ => TEXT_PROBE_LEN,
};
let mut head = [0u8; TEXT_PROBE_LEN];
let n = read_full(&mut file, head.get_mut(..want).unwrap_or_default())?;
pending.finish_with_head(head.get(..n).unwrap_or_default(), file)
}
pub fn tags_from_path(path: impl AsRef<Path>) -> Result<TagSet> {
tags_from_path_with(path, ExecPolicy::PythonCompat)
}
pub fn tags_from_path_with(path: impl AsRef<Path>, policy: ExecPolicy) -> Result<TagSet> {
let path = path.as_ref();
let md = fs::symlink_metadata(path).map_err(|e| IdentifyError::not_found(path, e))?;
tags_from_metadata(path, &md, policy)
}
pub fn tags_from_metadata(
path: impl AsRef<Path>,
metadata: &fs::Metadata,
policy: ExecPolicy,
) -> Result<TagSet> {
let path = path.as_ref();
let meta = Meta::from_metadata(metadata, policy);
match classify(meta, path_bytes(path)) {
Classification::Done(tags) => Ok(tags),
Classification::NeedsContent(pending) => {
finish_from_file(path, pending, Some(metadata.len()))
}
}
}
pub fn tags_from_meta(path: impl AsRef<Path>, meta: Meta) -> Result<TagSet> {
let path = path.as_ref();
match classify(meta, path_bytes(path)) {
Classification::Done(tags) => Ok(tags),
Classification::NeedsContent(pending) => finish_from_file(path, pending, None),
}
}
pub fn file_is_text(path: impl AsRef<Path>) -> Result<bool> {
let path = path.as_ref();
let md = fs::symlink_metadata(path).map_err(|e| IdentifyError::not_found(path, e))?;
let file_type = if md.file_type().is_symlink() {
fs::metadata(path)?.file_type()
} else {
md.file_type()
};
if FileKind::from_file_type(file_type) == FileKind::Other {
return Ok(false);
}
Ok(is_text(open_for_read(path)?)?)
}
pub fn parse_shebang_from_file(path: impl AsRef<Path>) -> Result<Vec<String>> {
parse_shebang_from_file_with(path, ExecPolicy::PythonCompat)
}
pub fn parse_shebang_from_file_with(
path: impl AsRef<Path>,
policy: ExecPolicy,
) -> Result<Vec<String>> {
let path = path.as_ref();
let md = fs::symlink_metadata(path).map_err(|e| IdentifyError::not_found(path, e))?;
let md = if md.file_type().is_symlink() {
match fs::metadata(path) {
Ok(target) => target,
Err(_) => return Ok(Vec::new()),
}
} else {
md
};
if !policy.is_executable(&md) || FileKind::from_file_type(md.file_type()) == FileKind::Other {
return Ok(Vec::new());
}
shebang_unless_einval(open_for_read(path))
}
fn shebang_unless_einval<R: Read>(file: io::Result<R>) -> Result<Vec<String>> {
match file.and_then(|file| parse_shebang_buf(&mut Source::new(&[], file))) {
Ok(cmd) => Ok(cmd),
Err(e) if is_python_einval(&e) => Ok(Vec::new()),
Err(e) => Err(e.into()),
}
}
#[cfg(unix)]
fn is_python_einval(err: &io::Error) -> bool {
err.raw_os_error() == Some(rustix::io::Errno::INVAL.raw_os_error())
}
#[cfg(windows)]
fn is_python_einval(err: &io::Error) -> bool {
err.raw_os_error()
.and_then(|code| u32::try_from(code).ok())
.is_some_and(crt_errno_is_einval)
}
#[cfg(not(any(unix, windows)))]
fn is_python_einval(err: &io::Error) -> bool {
err.raw_os_error().is_some() && err.kind() == io::ErrorKind::InvalidInput
}
#[cfg(any(windows, test))]
const fn crt_errno_is_einval(code: u32) -> bool {
!matches!(
code,
2..=11
| 15..=36
| 53
| 65
| 67
| 80
| 82
| 83
| 89
| 108
| 109
| 112
| 114
| 128..=130
| 132
| 145
| 158
| 161
| 164
| 167
| 183
| 188..=202
| 206
| 215
| 1816
)
}
#[derive(Clone, Copy, Debug)]
pub struct FileInfo<'a> {
pub filename: &'a str,
pub file_kind: FileKind,
pub is_executable: bool,
pub content: Option<&'a [u8]>,
}
#[must_use]
pub fn tags_from_info(info: &FileInfo<'_>) -> TagSet {
match classify(Meta::new(info.file_kind, info.is_executable), info.filename) {
Classification::Done(tags) => tags,
Classification::NeedsContent(pending) => match info.content {
Some(content) => pending.finish_bytes(content),
None => pending.tags(),
},
}
}
#[cfg(test)]
mod tests {
use super::*;
fn done(meta: Meta, name: &str) -> Option<String> {
match classify(meta, name) {
Classification::Done(tags) => Some(tags.to_string()),
Classification::NeedsContent(_) => None,
}
}
#[test]
fn classify_kinds() {
assert_eq!(
done(Meta::new(FileKind::Directory, true), "x.py").as_deref(),
Some("directory")
);
assert_eq!(
done(Meta::new(FileKind::Symlink, false), "x.py").as_deref(),
Some("symlink")
);
assert_eq!(
done(Meta::new(FileKind::Socket, false), "x").as_deref(),
Some("socket")
);
assert_eq!(
done(Meta::new(FileKind::Other, true), "fifo").as_deref(),
Some("binary, executable, file")
);
assert_eq!(
done(Meta::new(FileKind::Other, false), "x.txt").as_deref(),
Some("file, non-executable, plain-text, text")
);
assert_eq!(
done(Meta::regular(true), "x.py").as_deref(),
Some("executable, file, python, text")
);
assert_eq!(
done(Meta::regular(true), "x.sh").as_deref(),
Some("executable, file, shell, text")
);
}
#[test]
fn pending_rules() {
let Classification::NeedsContent(p) = classify(Meta::regular(false), "f.plist") else {
panic!("plist needs a binary check");
};
assert!(!p.needs_shebang() && p.needs_encoding());
assert_eq!(
p.finish_bytes(b"bplist00\xd1\x01").to_string(),
"binary, file, non-executable, plist"
);
assert_eq!(
p.finish_bytes(b"<?xml").to_string(),
"file, non-executable, plist, text"
);
let Classification::NeedsContent(p) = classify(Meta::regular(true), "run") else {
panic!("unknown executable needs content");
};
assert!(p.needs_shebang() && p.needs_encoding());
assert_eq!(
p.finish_bytes(b"#! \n").to_string(),
"executable, file, text"
);
assert_eq!(
p.finish_bytes(b"\x7fELF\x02").to_string(),
"binary, executable, file"
);
assert_eq!(
p.finish(&b"#!/bin/zsh\n"[..])
.map(|t| t.to_string())
.ok()
.as_deref(),
Some("executable, file, shell, text, zsh")
);
let mut long = b"#!/usr/bin/env ".to_vec();
long.extend(std::iter::repeat_n(b' ', 2000));
long.extend_from_slice(b"python3\n\0");
assert_eq!(
p.finish_bytes(&long).to_string(),
"executable, file, python, python3, text"
);
assert_eq!(
p.finish(&long[..]).map(|t| t.to_string()).ok().as_deref(),
Some("executable, file, python, python3, text")
);
}
struct Scripted {
data: &'static [u8],
error: Option<fn() -> io::Error>,
reads: usize,
}
impl Read for Scripted {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
self.reads += 1;
if self.data.is_empty() {
return self.error.map_or(Ok(0), |error| Err(error()));
}
let n = buf.len().min(self.data.len());
let (now, later) = self.data.split_at(n);
buf[..n].copy_from_slice(now);
self.data = later;
Ok(n)
}
}
fn scripted(data: &'static [u8], error: Option<fn() -> io::Error>) -> Scripted {
Scripted {
data,
error,
reads: 0,
}
}
#[cfg(unix)]
fn einval() -> io::Error {
io::Error::from_raw_os_error(rustix::io::Errno::INVAL.raw_os_error())
}
#[cfg(unix)]
fn eio() -> io::Error {
io::Error::from_raw_os_error(rustix::io::Errno::IO.raw_os_error())
}
#[cfg(windows)]
fn einval() -> io::Error {
io::Error::from_raw_os_error(1920) }
#[cfg(windows)]
fn eio() -> io::Error {
io::Error::from_raw_os_error(5) }
#[cfg(any(unix, windows))]
#[test]
fn shebang_einval_is_empty_on_open_and_on_read() {
assert_eq!(
shebang_unless_einval(Ok(scripted(b"#!/bin/sh\n", None))).ok(),
Some(vec!["/bin/sh".to_owned()])
);
assert_eq!(
shebang_unless_einval::<Scripted>(Err(einval())).ok(),
Some(vec![])
);
assert_eq!(
shebang_unless_einval(Ok(scripted(b"", Some(einval)))).ok(),
Some(vec![])
);
assert_eq!(
shebang_unless_einval(Ok(scripted(b"#!/bin/", Some(einval)))).ok(),
Some(vec![])
);
for result in [
shebang_unless_einval::<Scripted>(Err(eio())),
shebang_unless_einval(Ok(scripted(b"#!/bin/", Some(eio)))),
] {
let Err(IdentifyError::IoError { source }) = result else {
panic!("expected an I/O error, got {result:?}");
};
assert_eq!(source.raw_os_error(), eio().raw_os_error());
}
let custom = shebang_unless_einval(Ok(scripted(
b"",
Some(|| io::Error::new(io::ErrorKind::InvalidInput, "synthetic")),
)));
assert!(custom.is_err());
}
#[test]
fn crt_errno_mapping() {
for code in [1, 12, 13, 14, 87, 123, 131, 362, 1920, 4390, 0xffff_ffff] {
assert!(crt_errno_is_einval(code), "{code}");
}
for code in [
2, 3, 4, 5, 6, 8, 11, 15, 18, 19, 32, 33, 36, 53, 67, 80, 112, 145, 161, 183, 188, 202,
206, 1816,
] {
assert!(!crt_errno_is_einval(code), "{code}");
}
}
#[test]
fn size_hint_saves_the_eof_read() {
let Classification::NeedsContent(p) = classify(Meta::regular(false), "x.unknownext") else {
panic!("unknown name needs content");
};
let mut file = scripted(b"hello\n", None);
let tags = finish_sized(p, &mut file, Some(6)).ok();
assert_eq!(
tags.map(|t| t.to_string()).as_deref(),
Some("file, non-executable, text")
);
assert_eq!(file.reads, 1);
let mut file = scripted(b"hello\n", None);
assert!(finish_sized(p, &mut file, None).is_ok());
assert_eq!(file.reads, 2);
let mut file = scripted(b"\0", None);
let tags = finish_sized(p, &mut file, Some(0)).ok();
assert_eq!(
tags.map(|t| t.to_string()).as_deref(),
Some("binary, file, non-executable")
);
assert_eq!(file.reads, 2);
let Classification::NeedsContent(p) = classify(Meta::regular(true), "tool") else {
panic!("unknown executable needs content");
};
let mut file = scripted(b"#!/bin/zsh", None);
let tags = finish_sized(p, &mut file, Some(10)).ok();
assert_eq!(
tags.map(|t| t.to_string()).as_deref(),
Some("executable, file, shell, text, zsh")
);
}
#[test]
fn file_info_compat() {
let info = FileInfo {
filename: "noext",
file_kind: FileKind::Regular,
is_executable: false,
content: None,
};
assert_eq!(tags_from_info(&info).to_string(), "file, non-executable");
let info = FileInfo {
content: Some(b"text"),
..info
};
assert_eq!(
tags_from_info(&info).to_string(),
"file, non-executable, text"
);
}
#[test]
fn send_sync() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<Classification>();
assert_send_sync::<Pending>();
assert_send_sync::<Meta>();
assert_send_sync::<FileInfo<'static>>();
assert_send_sync::<IdentifyError>();
assert_send_sync::<crate::TagTable>();
}
}