1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424
// SPDX-FileCopyrightText: 2021-2022 Lynnesbian
// SPDX-License-Identifier: GPL-3.0-or-later
//! File handling - scanning, detecting MIME types, and so on.
use std::collections::{BTreeSet, HashMap};
use std::fs::File;
use std::io::{self, Read, Seek, SeekFrom};
use std::path::Path;
use std::str::FromStr;
use cfg_if::cfg_if;
use itertools::{Either, Itertools};
use log::{debug, error};
use mime::Mime;
use mime_guess::from_ext;
use once_cell::sync::Lazy;
use parking_lot::RwLock;
use walkdir::{DirEntry, WalkDir};
use crate::findings::{Findings, ScanError};
use crate::mime_db::MimeDb;
use crate::parameters::ScanOpts;
use crate::utils::APPLICATION_ZIP;
use crate::{String, MIMEDB};
/// Cache of MIME types and their associated extensions, used by [`mime_extension_lookup()`]
static MIMEXT: Lazy<RwLock<HashMap<String, Option<Vec<String>>>>> = Lazy::new(|| RwLock::new(HashMap::new()));
/// The number of bytes to read initially when identifying a file's MIME type. Used in the [`mime_type`] function.
///
/// Rather than reading the entire file all at once into a [`BUF_SIZE`] buffer, it tends to be faster to read a small
/// chunk of the file and trying to identify that, proceeding with the larger buffer if that fails. Many file formats
/// can be identified with the first few dozen bytes, so the "happy path" will likely be taken in the majority of cases.
pub const INITIAL_BUF_SIZE: usize = 128;
/// The number of bytes to read if the file couldn't be identified from its first [`INITIAL_BUF_SIZE`] bytes. Used in
/// the [`mime_type`] function.
pub const BUF_SIZE: usize = 8192;
/// A [`Mime`] representing the "application/x-ole-storage" MIME type.
static APPLICATION_X_OLE_STORAGE: Lazy<Mime> = Lazy::new(|| Mime::from_str("application/x-ole-storage").unwrap());
cfg_if! {
if #[cfg(windows)] {
/// Determines whether or not a file is hidden by checking its win32 file attributes.
pub fn is_hidden(entry: &DirEntry) -> bool {
use std::os::windows::prelude::*;
const FILE_ATTRIBUTE_HIDDEN: u32 = 0x2; // http://docs.microsoft.com/windows/win32/fileio/file-attribute-constants
std::fs::metadata(entry.path()) // try to get metadata for file
.map_or(
false, // if getting metadata/attributes fails, assume it's not hidden
|f| f.file_attributes() & FILE_ATTRIBUTE_HIDDEN > 0,
)
}
} else {
/// Determines whether or not a file is hidden by checking for a leading full stop.
pub fn is_hidden(entry: &DirEntry) -> bool {
entry
.file_name()
.to_str()
.map_or(false, |f| f.starts_with('.') && f != ".")
}
}
}
/// Returns `true` if a file matches the given criteria. This means checking whether the file's extension appears in
/// `exts` (if specified), potentially skipping over hidden files, and so on.
pub fn wanted_file(
entry: &DirEntry,
exts: Option<&BTreeSet<&str>>,
exclude: Option<&BTreeSet<&str>>,
scan_opts: &ScanOpts,
) -> bool {
if entry.depth() == 0 {
// the root directory should always be scanned.
return true;
}
if !scan_opts.hidden && is_hidden(entry) {
// skip hidden files and directories. this check is performed first because it's very lightweight.
return false;
}
if entry.file_type().is_dir() {
// always allow directories - there's no point doing file extension matching on something that isn't a file.
return true;
}
if let Some(ext) = entry.path().extension() {
// file has extension - discard invalid UTF-8 and normalise it to lowercase.
let ext = ext.to_string_lossy().to_lowercase();
let ext = ext.as_str();
if scan_opts.ignore_unknown_exts && from_ext(ext).is_empty() {
// unknown extension, skip.
return false;
}
if let Some(exts) = exts {
// only scan if the file has one of the specified extensions.
exts.contains(&ext)
} else {
// no extensions specified - the file should be scanned unless its extension is on the exclude list.
exclude.map_or(true, |exclude| !exclude.contains(&ext))
}
} else {
// no file extension
scan_opts.extensionless
}
}
/// Inspects the given entry, returning a [`Findings`] on success and a [`ScanError`] on failure.
///
/// In the event of an IO error, the returned [`ScanError`] will be of type [`ScanError::File`]. Otherwise, a
/// [`ScanError::Mime`] will be returned, meaning that the file was scanned successfully, but a MIME type could not be
/// determined.
pub fn scan_file(entry: &DirEntry, canonical_paths: bool) -> Result<Findings, ScanError> {
let path = entry.path();
// try to determine MIME type for this entry
let result = match mime_type(&*MIMEDB, path) {
// an error occurred while trying to read the file
Err(_) => return Err(ScanError::File(path)),
// the file was read successfully, but we were unable to determine its MIME type
Ok(None) => return Err(ScanError::Mime(path)),
// a MIME type was found!
Ok(Some(result)) => result,
};
// determine whether or not the file's current extension is valid
let valid = if let Some(entry_ext) = path.extension() {
// discard invalid UTF-8 and convert to lowercase. all extensions in both backend's databases are lowercase
// ascii, so this assumption is fine.
let entry_ext = entry_ext.to_string_lossy().to_lowercase();
// if the file has any of these extensions, it is probably either:
// - a copy of another file renamed for backup purposes (e.g. a word processor might save by renaming "my.doc" to
// "my.doc.bak", then creating "my.doc", leaving the backup for safekeeping), which shouldn't be renamed so as
// not to break the backup program
// - a partially downloaded file, which shouldn't be renamed to avoid corrupting it and blocking the downloader
// from resuming
if ["bak", "backup", "filepart", "part", "crdownload"]
.iter()
.any(|ext| ext == &entry_ext)
{
true
} else {
// otherwise, check to see whether there's a known extension for this file type
// retrieve set of known extensions for the given MIME type
let known_exts = mime_extension_lookup(result.essence_str().into());
match known_exts {
// there is a known set of extensions for this MIME type - is entry_ext in the given set?
Some(e) => e.contains(&entry_ext.into()),
// there is no known set of extensions for this MIME type :(
None => false,
}
}
} else {
// this file has no extension
false
};
let path = if canonical_paths {
match std::fs::canonicalize(path) {
Ok(path) => path,
Err(_) => return Err(ScanError::File(entry.path())),
}
} else {
path.to_path_buf() // :c
};
Ok(Findings {
file: path,
valid,
mime: result,
})
}
/// Takes a slice of [`DirEntry`]s and calls [`scan_file`] on each one, returning the results in a vector.
pub fn scan_from_walkdir(
entries: &[DirEntry],
canonical_paths: bool,
use_threads: bool,
) -> (Vec<Findings>, Vec<ScanError>) {
cfg_if! {
if #[cfg(feature = "multi-threaded")] {
use rayon::prelude::*;
const CHUNKS: usize = 32;
if use_threads && entries.len() > CHUNKS {
// split the entries into chunks of 32, and iterate over each chunk of entries in a separate thread
return entries
.par_chunks(CHUNKS)
.flat_map_iter(|chunk| {
chunk
.iter() // iter over the chunk, which is a slice of DirEntry structs
.map(|entry| scan_file(entry, canonical_paths))
.collect::<Vec<_>>()
}).partition_map(|result| match result {
Ok(f) => Either::Left(f),
Err(e) => Either::Right(e),
});
}
} else {
// should always be false when multi-threading is disabled at compile time
assert!(!use_threads)
}
}
// if we end up here, either
// - there were less than CHUNKS files to scan, or
// - the user specified that only one thread should be used, by specifying `-j 1`
// - fif was compiled without the `multi-threading` feature
entries
.iter()
.partition_map(|entry: &DirEntry| match scan_file(entry, canonical_paths) {
Ok(f) => Either::Left(f),
Err(e) => Either::Right(e),
})
}
/// Scans a given directory with [`WalkDir`], filters with [`wanted_file`], checks for errors, and returns a vector of
/// [`DirEntry`]s.
pub fn scan_directory(
dirs: &Path,
exts: Option<&BTreeSet<&str>>,
exclude: Option<&BTreeSet<&str>>,
scan_opts: &ScanOpts,
) -> Option<Vec<DirEntry>> {
let stepper = WalkDir::new(dirs).follow_links(scan_opts.follow_symlinks).into_iter();
let mut probably_fatal_error = false;
let entries: Vec<DirEntry> = stepper
.filter_entry(|e| wanted_file(e, exts, exclude, scan_opts)) // filter out unwanted files
.filter_map(|e| {
if let Err(err) = &e {
debug!("uh oh spaghettio!! {:#?}", e);
// log errors to stdout, and remove them from the iterator
let path = err.path().map_or("General error".into(), Path::to_string_lossy);
if err.depth() == 0 {
// if something goes wrong while trying to read the root directory, we're probably not going to get much done
probably_fatal_error = true;
}
// TODO: is there a way to just say `map_or(x, |y| y).thing()` instead of `map_or(x.thing(), |y| y.thing())`?
// i don't care whether i'm returning a walkdir error or an io error, i just care about whether or not it
// implements ToString (which they both do). map_or doesn't work on trait objects though :(
error!("{}: {}", path, err.io_error().map_or(err.to_string(), |e| e.to_string()));
return None;
}
e.ok()
})
// remove directories from the final list
.filter(|e| !e.file_type().is_dir())
// if fif is invoked without `-f` on a symlinked directory, it will recurse into the symlink (as desired) and ignore
// any symlinks inside the symlinked root directory. however, the root directory will still be added to `entries` as
// if it were a file to be scanned, and `scan_file` will fail to scan it, adding "Failed to read ~/whatever" to the
// output. to avoid this, we can remove all symlinks from `entries` if `-f` is not set. i know this is kind of
// confusing, but it's honestly kind of hard to explain... maybe a screenshot is better:
// https://i.imgur.com/DYG7jlB.png
// adding the symlink filter removes the line that's being pointed to in the image. 0u0
.filter(|e| scan_opts.follow_symlinks || !e.file_type().is_symlink())
.collect();
if probably_fatal_error {
None
} else {
Some(entries)
}
}
/// Tries to identify the MIME type of a file from a given path.
pub fn mime_type<T: MimeDb>(db: &T, path: &Path) -> io::Result<Option<Mime>> {
let mut buffer = [0; INITIAL_BUF_SIZE];
let mut file = File::open(path)?;
// read a small amount to start with
let mut read = io::Result::Ok(0);
for _ in 0..3 {
// try to read the file up to 3 times, retrying if interrupted, bailing otherwise
file.seek(SeekFrom::Start(0))?;
read = file.read(&mut buffer);
match read {
Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
Err(_) | Ok(_) => break,
}
}
let read = read?;
let r = db.get_type(&buffer);
if read < INITIAL_BUF_SIZE {
// the file is smaller than INITIAL_BUF_SIZE - there's no point reading it again
return Ok(r);
}
let r = r.filter(|mime|
// some MIME types should be investigated further, reading up to BUF_SIZE even if they've been determined already
// one such type is XML - there's many more specific types that can be determined by reading further (such as SVG)
mime != &mime::TEXT_XML
// another is ZIP - many file formats (DOCX, ODT, JAR...) are just ZIP files with particular data structures.
// determining that a file is in one of the MS office formats in particular requires looking quite far into the
// file.
&& mime != &*APPLICATION_ZIP
// doc/ppt/xls files are a subset of what's known as an "OLE2 compound document storage", at least according to
// shared-mime-info. if a pre-OOXML era MS office file is scanned and identified as x-ole-storage, reading further
// will allow it to be detected correctly as the appropriate filetype.
&& mime != &*APPLICATION_X_OLE_STORAGE);
if r.is_some() {
return Ok(r);
}
// attempt to read up to BUF_SIZE bytes of the file.
let mut buffer = [0; BUF_SIZE];
file.seek(SeekFrom::Start(0))?;
_ = file.read(&mut buffer)?;
Ok(db.get_type(&buffer))
}
/// Returns a list of known extensions for this MIME type, if any.
/// This function uses the [`Mime`]'s "essence" rather than the [`Mime`] itself - [`mime_guess::get_mime_extensions`]
/// ignores the type suffix, treating "image/svg+xml" as "image/svg", and thus fails to find any extensions. Passing the
/// `essence_str` (which includes the suffix) fixes this.
pub fn mime_extension_lookup(essence: String) -> Option<Vec<String>> {
if let Some(exts) = MIMEXT.read().get(&essence) {
return exts.clone();
}
let mut exts = mime_guess::get_mime_extensions_str(essence.as_str());
if exts.is_none() {
// no matches :c
// mime_guess' database isn't exactly perfect... there are a lot of times where the db will return "some/x-thing"
// but mime_guess only understands "some/thing", or vice-versa.
// so, if there appear to be no extensions, try replacing "some/x-thing" with "some/thing", or "some/thing" with
// "some/x-thing".
if essence.contains("/x-") {
// replace e.g. "application/x-gzip" with "application/gzip"
exts = mime_guess::get_mime_extensions_str(&essence.replace("/x-", "/"));
} else {
// replace e.g. "video/mp2t" with "video/x-mp2t"
exts = mime_guess::get_mime_extensions_str(&essence.replace('/', "/x-"));
}
}
let exts = match exts {
Some(exts) => {
let possible_exts: Vec<String> = exts.iter().map(|e| String::from(*e)).collect();
Some(if essence == mime::IMAGE_JPEG.essence_str() {
// possible_exts starts with "jpe", because it's alphabetically before "jpeg" and "jpg". however, jpg/jpeg are
// far more common than jpe, so it makes sense to suggest one of those rather than jpe. to do this, we can
// add "jpg" to the start of the possible_exts list, ensuring that it will be the extension suggested by fif.
[vec![String::from("jpg")], possible_exts].concat()
} else if essence == mime::TEXT_XML.essence_str() || essence == "application/xml" {
// a somewhat similar case arises with XML files - the first suggested extension is "asa", when it should
// (in my opinion) be "xml".
// there's also another problem: SVG files can easily be misidentified as XML files, because they usually
// *are* valid XML - the more whitespace and comments an SVG file begins with, the more bytes must be read
// before it's possible to determine that it's an SVG rather than an XML file. to "fix" this, we can add "svg"
// as a valid extension for XML files, ensuring that SVG files misidentified as XML will still be considered
// to have valid extensions.
// TODO: if a file is detected as application/xml, but it has an extension like "xht" which corresponds to
// "application/xhtml+xml", let it through - in other words, if it's identified as application/xml, but its
// extension is classed as application/*+xml, consider it OK
[vec![String::from("xml"), String::from("svg")], possible_exts].concat()
} else if essence == "application/msword" {
// classic office files considered harmful
vec![String::from("doc"), String::from("xls"), String::from("ppt")]
} else if essence == "application/zip" {
// both backends seem to be unable to consistently detect OOXML files, so they should be considered valid
// extensions for zip files to prevent them being erroneously renamed.
// additionally, there are various popular formats that are just renamed zip files, such as android's apk
// format, that also shouldn't be renamed.
[
vec![
String::from("zip"),
String::from("docx"),
String::from("xlsx"),
String::from("pptx"),
String::from("apk"),
String::from("ipa"),
String::from("docbook"),
String::from("kdenlive"),
String::from("vcpkg"),
String::from("nupkg"),
String::from("whl"),
String::from("xpi"),
],
possible_exts,
]
.concat()
} else if essence == "application/x-ms-dos-executable" {
// .dll, .exe, .scr, etc. files are given the same MIME type, and aren't really distinguishable from each other
// ... but you definitely don't want to rename one to the other!
[
vec![
String::from("exe"),
String::from("dll"),
String::from("scr"),
String::from("com"),
String::from("dll16"),
String::from("drv"),
String::from("drv16"),
String::from("cpl"),
String::from("msstyles"),
String::from("sys"),
],
possible_exts,
]
.concat()
} else {
possible_exts
})
}
None => None,
};
MIMEXT.write().insert(essence, exts.clone());
exts
}