use crate::fs_beneath::{entry_metadata_at, open_directory_beneath, open_file_at, read_link_at};
use crate::git_system_config::system_attributes_for;
use crate::repo_read_config_list::list_config;
use crate::repo_read_config_scan::ScopeEnvironment;
use crate::repo_read_config_text::{ConfigEntry, last_entry, parse_bool};
use crate::repo_read_layout::ObjectFormat;
use crate::subvolume_sha::{sha1, sha256};
use crate::worktree_attributes::{AttributeLine, CONVERTING_ATTRIBUTES, attribute_name, may_apply, parse_attribute_lines};
use crate::worktree_clean::{StatEntry, StatVerdict, metadata_verdict};
use std::collections::{HashMap, HashSet, VecDeque};
use std::ffi::{OsStr, OsString};
use std::io::Read;
use std::os::unix::ffi::{OsStrExt, OsStringExt};
use std::os::unix::fs::{MetadataExt, OpenOptionsExt};
use std::path::{Path, PathBuf};
pub const CONTENT_LIMIT: u64 = 32 << 20;
pub const ATTRIBUTES_LIMIT: u64 = 4 << 20;
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum SystemAttributes {
Disabled,
File(PathBuf),
Unknown,
}
#[derive(Clone, Debug)]
pub struct ConversionEnvironment {
pub scope: ScopeEnvironment,
pub system_attributes: SystemAttributes,
pub alternate_attribute_source: Option<String>,
pub variables: Vec<(OsString, OsString)>,
}
impl ConversionEnvironment {
pub fn from_variables(variables: &[(OsString, OsString)]) -> ConversionEnvironment {
let selected: Option<String> = variables.iter().any(|(name, _value): &(OsString, OsString)| { return name == "GIT_ATTR_SOURCE"; }).then(|| { return String::from("GIT_ATTR_SOURCE is set"); });
return ConversionEnvironment { scope: ScopeEnvironment::from_variables(variables), system_attributes: system_attributes_from(variables), alternate_attribute_source: selected, variables: variables.to_vec() };
}
}
pub fn system_attributes_from(variables: &[(OsString, OsString)]) -> SystemAttributes {
let switch: Option<&OsString> = variables.iter().rev().find(|(name, _value): &&(OsString, OsString)| { return name == "GIT_ATTR_NOSYSTEM"; }).map(|(_name, value): &(OsString, OsString)| { return value; });
let disabled: Option<bool> = match switch {
None => Some(false),
Some(value) if value.is_empty() => Some(false),
Some(value) => parse_bool(Some(value.as_bytes())),
};
return match disabled {
Some(true) => SystemAttributes::Disabled,
Some(false) => system_attributes_for(variables).map_or(SystemAttributes::Unknown, SystemAttributes::File),
None => SystemAttributes::Unknown,
};
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct AttributeFile {
label: String,
base: Vec<u8>,
lines: Vec<AttributeLine>,
}
#[derive(Clone, Debug)]
struct FixedSources {
files: Vec<AttributeFile>,
macros: HashSet<Vec<u8>>,
}
fn blob_name(data: &[u8], format: ObjectFormat) -> Vec<u8> {
let mut framed: Vec<u8> = format!("blob {}\0", data.len()).into_bytes();
framed.extend_from_slice(data);
return match format {
ObjectFormat::Sha1 => sha1(framed.as_slice()).to_vec(),
ObjectFormat::Sha256 => sha256(framed.as_slice()).to_vec(),
};
}
pub fn read_at_most(reader: impl Read, limit: u64) -> std::io::Result<Option<Vec<u8>>> {
let mut data: Vec<u8> = Vec::new();
reader.take(limit + 1).read_to_end(&mut data)?;
return Ok(if data.len() as u64 > limit { None } else { Some(data) });
}
fn same_state(first: &std::fs::Metadata, second: &std::fs::Metadata) -> bool {
return first.dev() == second.dev()
&& first.ino() == second.ino()
&& first.size() == second.size()
&& first.mode() == second.mode()
&& first.mtime() == second.mtime()
&& first.mtime_nsec() == second.mtime_nsec()
&& first.ctime() == second.ctime()
&& first.ctime_nsec() == second.ctime_nsec();
}
fn converts(token: &[u8], macros: &HashSet<Vec<u8>>) -> bool {
let name: &[u8] = attribute_name(token);
return CONVERTING_ATTRIBUTES.contains(&name) || macros.contains(name);
}
fn converting_macros(files: &[AttributeFile]) -> HashSet<Vec<u8>> {
let mut mentioned_by: HashMap<Vec<u8>, Vec<Vec<u8>>> = HashMap::new();
for line in files.iter().flat_map(|file: &AttributeFile| { return file.lines.iter(); }).filter(|line: &&AttributeLine| { return line.is_macro; }) {
for token in &line.tokens {
mentioned_by.entry(attribute_name(token.as_slice()).to_vec()).or_default().push(line.subject.clone());
}
}
let mut known: HashSet<Vec<u8>> = HashSet::new();
let mut queue: VecDeque<Vec<u8>> = CONVERTING_ATTRIBUTES.iter().map(|name: &&[u8]| { return name.to_vec(); }).collect();
while let Some(name) = queue.pop_front() {
for macro_name in mentioned_by.get(&name).into_iter().flatten() {
if known.insert(macro_name.clone()) {
queue.push_back(macro_name.clone());
}
}
}
return known;
}
pub struct ContentJudge<'a> {
root: &'a std::fs::File,
git_dir: &'a Path,
common_dir: &'a Path,
environment: &'a ConversionEnvironment,
format: ObjectFormat,
fixed: Option<Result<FixedSources, String>>,
directories: HashMap<Vec<u8>, Option<AttributeFile>>,
}
pub struct ReadSeams<'a> {
pub before_open: &'a mut dyn FnMut(),
pub before_read: &'a mut dyn FnMut(),
pub after_read: &'a mut dyn FnMut(),
}
pub struct Place<'a> {
pub directory: &'a std::fs::File,
pub parents: &'a [OsString],
pub name: &'a OsStr,
}
impl<'a> ContentJudge<'a> {
pub fn new(root: &'a std::fs::File, git_dir: &'a Path, common_dir: &'a Path, environment: &'a ConversionEnvironment, format: ObjectFormat) -> ContentJudge<'a> {
return ContentJudge { root, git_dir, common_dir, environment, format, fixed: None, directories: HashMap::new() };
}
fn read_outside(path: &Path, label: String) -> Result<Option<AttributeFile>, String> {
let file = match std::fs::OpenOptions::new().read(true).custom_flags(libc::O_NONBLOCK | libc::O_CLOEXEC).open(path) {
Ok(opened) => opened,
Err(error) if matches!(error.kind(), std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory) => return Ok(None),
Err(error) => return Err(format!("the attributes file {} cannot be read ({error})", path.display())),
};
return Self::parse_open(file, label, Vec::new());
}
fn read_in(directory: &std::fs::File, label: String, base: Vec<u8>) -> Result<Option<AttributeFile>, String> {
let file = match open_file_at(directory, OsStr::new(".gitattributes")) {
Ok(opened) => opened,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(error) => return Err(format!("{label} cannot be read without following a link ({error}), so the attributes it sets cannot be ruled out")),
};
return Self::parse_open(file, label, base);
}
fn parse_open(file: std::fs::File, label: String, base: Vec<u8>) -> Result<Option<AttributeFile>, String> {
let metadata: std::fs::Metadata = file.metadata().map_err(|error: std::io::Error| { return format!("{label} cannot be examined ({error})"); })?;
if !metadata.file_type().is_file() {
return Err(format!("{label} is not a plain file (a named pipe, a device or a directory), so the attributes it sets cannot be ruled out"));
}
let bytes: Vec<u8> = read_at_most(&file, ATTRIBUTES_LIMIT)
.map_err(|error: std::io::Error| { return format!("{label} cannot be read ({error})"); })?
.ok_or_else(|| { return format!("{label} is larger than {ATTRIBUTES_LIMIT} bytes, so the attributes it sets cannot be ruled out"); })?;
return Ok(Some(AttributeFile { label, base, lines: parse_attribute_lines(bytes.as_slice()) }));
}
fn config_entries(&self) -> Result<Vec<ConfigEntry>, String> {
let listed = list_config(self.common_dir, self.git_dir, &self.environment.scope, self.environment.variables.as_slice())
.ok_or_else(|| { return String::from("the Git configuration cannot be read without running Git, so a global attributes file cannot be ruled out"); })?;
return Ok(listed.into_iter().map(|item| { return item.entry; }).collect());
}
fn global_attributes_path(&self, entries: &[ConfigEntry]) -> Result<Option<PathBuf>, String> {
if let Some(entry) = last_entry(entries, "core.attributesfile") {
let value: &[u8] = entry.value.as_deref().ok_or_else(|| { return String::from("core.attributesFile has no value"); })?;
if let Some(rest) = value.strip_prefix(b"~/") {
let home: &Path = self.environment.scope.home.as_deref().ok_or_else(|| { return String::from("core.attributesFile starts with ~/ but HOME is not set"); })?;
return Ok(Some(home.join(OsStr::from_bytes(rest))));
}
let path: PathBuf = PathBuf::from(OsStr::from_bytes(value));
if !path.is_absolute() {
return Err(format!("core.attributesFile is {:?}, which is not an absolute path", String::from_utf8_lossy(value)));
}
return Ok(Some(path));
}
return Ok(match (&self.environment.scope.xdg, &self.environment.scope.home) {
(Some(directory), _) => Some(directory.join("git/attributes")),
(None, Some(home)) => Some(home.join(".config/git/attributes")),
(None, None) => None,
});
}
fn load_fixed(&self) -> Result<FixedSources, String> {
let unmodelled = |reason: &str| -> String { return format!("{reason}, so Git takes its attributes from a tree and not from the files read here, and what it sets cannot be ruled out"); };
if let Some(reason) = self.environment.alternate_attribute_source.as_deref() {
return Err(unmodelled(reason));
}
let entries: Vec<ConfigEntry> = self.config_entries()?;
if last_entry(entries.as_slice(), "attr.tree").is_some() {
return Err(unmodelled("attr.tree is set"));
}
let mut files: Vec<AttributeFile> = Vec::new();
if let Some(root_file) = Self::read_in(self.root, String::from(".gitattributes"), Vec::new())? {
files.push(root_file);
}
let mut outside: Vec<(PathBuf, String)> = vec![(self.common_dir.join("info/attributes"), String::from("info/attributes"))];
match &self.environment.system_attributes {
SystemAttributes::Disabled => {}
SystemAttributes::File(path) => outside.push((path.clone(), path.display().to_string())),
SystemAttributes::Unknown => return Err(String::from("the location of the system attributes file cannot be established without running Git, so what it sets cannot be ruled out")),
}
if let Some(path) = self.global_attributes_path(entries.as_slice())? {
outside.push((path.clone(), path.display().to_string()));
}
for (path, label) in outside {
if let Some(file) = Self::read_outside(path.as_path(), label)? {
files.push(file);
}
}
let macros: HashSet<Vec<u8>> = converting_macros(files.as_slice());
return Ok(FixedSources { files, macros });
}
fn directory_file(&mut self, parents: &[OsString]) -> Result<Option<AttributeFile>, String> {
let key: Vec<u8> = parents.iter().map(|name: &OsString| { return name.as_bytes().to_vec(); }).collect::<Vec<Vec<u8>>>().join(&b'/');
if !self.directories.contains_key(&key) {
let directory = open_directory_beneath(self.root, parents).map_err(|error: std::io::Error| { return format!("{} cannot be opened below the worktree ({error})", String::from_utf8_lossy(key.as_slice())); })?;
let label: String = format!("{}/.gitattributes", String::from_utf8_lossy(key.as_slice()));
let file: Option<AttributeFile> = Self::read_in(&directory, label, key.clone())?;
self.directories.insert(key.clone(), file);
}
return Ok(self.directories.get(&key).cloned().flatten());
}
fn converting_attribute(&mut self, path: &[u8], parents: &[OsString]) -> Result<Option<(String, String)>, String> {
if self.fixed.is_none() {
self.fixed = Some(self.load_fixed());
}
let fixed: FixedSources = self.fixed.clone().ok_or_else(|| { return String::from("the attributes were not loaded"); })??;
let mut files: Vec<AttributeFile> = fixed.files;
for depth in 1..=parents.len() {
if let Some(file) = self.directory_file(&parents[..depth])? {
files.push(file);
}
}
for file in &files {
let below: bool = file.base.is_empty() || path.strip_prefix(file.base.as_slice()).is_some_and(|rest: &[u8]| { return rest.starts_with(b"/"); });
if below && file.lines.iter().any(|line: &AttributeLine| { return line.quoted; }) {
return Ok(Some((String::from("a quoted line"), file.label.clone())));
}
for line in file.lines.iter().filter(|line: &&AttributeLine| { return !line.is_macro; }) {
let Some(token) = line.tokens.iter().find(|token: &&Vec<u8>| { return converts(token.as_slice(), &fixed.macros); }) else { continue };
if may_apply(line.subject.as_slice(), file.base.as_slice(), path) {
return Ok(Some((String::from_utf8_lossy(token.as_slice()).into_owned(), file.label.clone())));
}
}
}
return Ok(None);
}
fn read_stable(place: &Place<'_>, entry: &StatEntry, index_seconds: u32, shown: &str, seams: &mut ReadSeams<'_>) -> Result<Vec<u8>, String> {
let changed = || -> String { return format!("{shown} changed while it was examined, so it cannot be proven equal to the committed content"); };
let named: std::fs::Metadata = entry_metadata_at(place.directory, place.name).map_err(|error: std::io::Error| { return format!("{shown} cannot be examined ({error})"); })?;
if let StatVerdict::Differs(reason) = metadata_verdict(entry, &named, index_seconds) {
return Err(reason);
}
if named.file_type().is_symlink() {
let target: PathBuf = read_link_at(place.directory, place.name).map_err(|error: std::io::Error| { return format!("{shown} cannot be read ({error})"); })?;
let again: std::fs::Metadata = entry_metadata_at(place.directory, place.name).map_err(|_error: std::io::Error| { return changed(); })?;
if !same_state(&named, &again) {
return Err(changed());
}
return Ok(target.into_os_string().into_vec());
}
(seams.before_open)();
let file: std::fs::File = open_file_at(place.directory, place.name).map_err(|error: std::io::Error| { return format!("{shown} cannot be opened without following a link ({error})"); })?;
let held: std::fs::Metadata = file.metadata().map_err(|error: std::io::Error| { return format!("{shown} cannot be examined ({error})"); })?;
if held.dev() != named.dev() || held.ino() != named.ino() {
return Err(changed());
}
if let StatVerdict::Differs(reason) = metadata_verdict(entry, &held, index_seconds) {
return Err(reason);
}
let too_large = || -> String { return format!("{shown} is larger than {CONTENT_LIMIT} bytes, which is more than is read to compare it by content"); };
if held.len() > CONTENT_LIMIT {
return Err(too_large());
}
(seams.before_read)();
let data: Vec<u8> = read_at_most(&file, CONTENT_LIMIT).map_err(|error: std::io::Error| { return format!("{shown} cannot be read ({error})"); })?.ok_or_else(too_large)?;
(seams.after_read)();
let after: std::fs::Metadata = file.metadata().map_err(|_error: std::io::Error| { return changed(); })?;
let by_name: std::fs::Metadata = entry_metadata_at(place.directory, place.name).map_err(|_error: std::io::Error| { return changed(); })?;
if !same_state(&held, &after) || !same_state(&held, &by_name) {
return Err(changed());
}
return Ok(data);
}
pub fn agrees(&mut self, place: &Place<'_>, entry: &StatEntry, id: &[u8], index_seconds: u32) -> Result<(), String> {
return self.agrees_seamed(place, entry, id, index_seconds, &mut ReadSeams { before_open: &mut || {}, before_read: &mut || {}, after_read: &mut || {} });
}
pub fn agrees_seamed(&mut self, place: &Place<'_>, entry: &StatEntry, id: &[u8], index_seconds: u32, seams: &mut ReadSeams<'_>) -> Result<(), String> {
let shown: String = String::from_utf8_lossy(entry.path.as_slice()).into_owned();
if let Some((token, label)) = self.converting_attribute(entry.path.as_slice(), place.parents)? {
return Err(if token == "a quoted line" {
format!("{shown} cannot be compared by content: {label} has a quoted line, whose pattern and attributes are not modelled, so what it sets cannot be ruled out")
} else {
format!("{shown} cannot be compared by content: the attribute {token} from {label} may change its bytes on the way into the index")
});
}
let data: Vec<u8> = Self::read_stable(place, entry, index_seconds, shown.as_str(), seams)?;
if blob_name(data.as_slice(), self.format) != id {
return Err(format!("{shown} differs from the committed content (its recorded size and times could not vouch for it, so its content was compared)"));
}
if data.contains(&b'\r') {
return Err(format!("{shown} cannot be compared by content: it contains a carriage return, so a line-ending conversion may change its bytes on the way into the index"));
}
return Ok(());
}
}
#[cfg(test)]
#[path = "worktree_clean_content_tests.rs"]
mod tests;
#[cfg(test)]
#[path = "worktree_clean_content_safety_tests.rs"]
mod safety_tests;