use std::collections::BTreeSet;
use std::fmt;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use dashmap::mapref::entry::Entry;
pub use directory::{
DirectoryListing, DirectoryListingError, directory_listing, system_path_to_directory,
};
pub use file_root::{FileRoot, FileRootKind};
pub use path::FilePath;
use ruff_notebook::{Notebook, NotebookError};
use ruff_python_ast::PySourceType;
use ruff_text_size::{Ranged, TextRange};
use salsa::plumbing::AsId;
use salsa::{Durability, Setter};
use crate::diagnostic::{Span, UnifiedFile};
use crate::file_revision::FileRevision;
use crate::files::file_root::FileRoots;
use crate::files::private::FileStatus;
use crate::source::SourceText;
use crate::system::{
SystemPath, SystemPathBuf, SystemVirtualPath, SystemVirtualPathBuf, deduplicate_nested_paths,
};
use crate::vendored::{VendoredPath, VendoredPathBuf};
use crate::{Db, FxDashMap, vendored};
mod directory;
mod file_root;
mod path;
#[inline]
pub fn system_path_to_file(db: &dyn Db, path: impl AsRef<SystemPath>) -> Result<File, FileError> {
let file = db.files().system(db, path.as_ref());
match file.status(db) {
FileStatus::Exists => Ok(file),
FileStatus::IsADirectory => Err(FileError::IsADirectory),
FileStatus::NotFound => Err(FileError::NotFound),
}
}
#[inline]
pub fn vendored_path_to_file(
db: &dyn Db,
path: impl AsRef<VendoredPath>,
) -> Result<File, FileError> {
db.files().vendored(db, path.as_ref())
}
#[derive(Default, Clone)]
pub struct Files {
inner: Arc<FilesInner>,
}
#[derive(Default)]
struct FilesInner {
frozen: AtomicBool,
system_by_path: FxDashMap<SystemPathBuf, File>,
system_virtual_by_path: FxDashMap<SystemVirtualPathBuf, VirtualFile>,
vendored_by_path: FxDashMap<VendoredPathBuf, File>,
roots: std::sync::RwLock<FileRoots>,
}
impl Files {
pub fn freeze(&self) {
self.inner.frozen.store(true, Ordering::Relaxed);
}
fn input_durability(&self, default: Durability) -> Durability {
if self.inner.frozen.load(Ordering::Relaxed) {
Durability::NEVER_CHANGE
} else {
default
}
}
fn system(&self, db: &dyn Db, path: &SystemPath) -> File {
if path.is_absolute()
&& let Some(file) = self.inner.system_by_path.get(path)
{
return *file;
}
let absolute = SystemPath::absolute(path, db.system().current_directory());
if let Some(file) = self.inner.system_by_path.get(absolute.as_path()) {
return *file;
}
*self
.inner
.system_by_path
.entry(absolute.clone())
.or_insert_with(|| {
let metadata = db.system().path_metadata(path);
tracing::trace!("Adding file '{absolute}'");
let durability = self.input_durability(
self.root(db, &absolute)
.map_or(Durability::default(), |root| root.durability(db)),
);
let builder = File::builder(FilePath::from(absolute))
.durability(durability)
.path_durability(Durability::NEVER_CHANGE);
let builder = match metadata {
Ok(metadata) if metadata.file_type().is_file() => builder
.permissions(metadata.permissions())
.revision(metadata.revision()),
Ok(metadata) if metadata.file_type().is_directory() => builder
.durability(Durability::MEDIUM.max(durability))
.status(FileStatus::IsADirectory)
.permissions(metadata.permissions())
.revision(metadata.revision()),
_ => builder
.status(FileStatus::NotFound)
.status_durability(Durability::MEDIUM.max(durability)),
};
builder.new(db)
})
}
pub fn try_system(&self, db: &dyn Db, path: &SystemPath) -> Option<File> {
if path.is_absolute()
&& let Some(file) = self.inner.system_by_path.get(path)
{
return Some(*file);
}
let absolute = SystemPath::absolute(path, db.system().current_directory());
self.inner
.system_by_path
.get(&absolute)
.map(|entry| *entry.value())
}
fn vendored(&self, db: &dyn Db, path: &VendoredPath) -> Result<File, FileError> {
if let Some(file) = self.inner.vendored_by_path.get(path) {
return Ok(*file);
}
let file = match self.inner.vendored_by_path.entry(path.to_path_buf()) {
Entry::Occupied(entry) => *entry.get(),
Entry::Vacant(entry) => {
let metadata = match db.vendored().metadata(path) {
Ok(metadata) => match metadata.kind() {
vendored::FileType::File => metadata,
vendored::FileType::Directory => return Err(FileError::IsADirectory),
},
Err(_) => return Err(FileError::NotFound),
};
tracing::trace!("Adding vendored file `{}`", path);
let file = File::builder(FilePath::from(path))
.permissions(Some(0o444))
.revision(metadata.revision())
.durability(Durability::NEVER_CHANGE)
.new(db);
entry.insert(file);
file
}
};
Ok(file)
}
pub fn virtual_file(&self, db: &dyn Db, path: &SystemVirtualPath) -> VirtualFile {
tracing::trace!("Adding virtual file {}", path);
let virtual_file = VirtualFile(
File::builder(FilePath::from(path))
.durability(self.input_durability(Durability::LOW))
.path_durability(Durability::NEVER_CHANGE)
.status(FileStatus::Exists)
.revision(FileRevision::zero())
.permissions(None)
.permissions_durability(Durability::NEVER_CHANGE)
.new(db),
);
self.inner
.system_virtual_by_path
.insert(path.to_path_buf(), virtual_file);
virtual_file
}
pub fn try_virtual_file(&self, path: &SystemVirtualPath) -> Option<VirtualFile> {
self.inner
.system_virtual_by_path
.get(path)
.map(|entry| *entry.value())
}
pub fn root(&self, db: &dyn Db, path: &SystemPath) -> Option<FileRoot> {
let roots = self.inner.roots.read().unwrap();
let absolute = SystemPath::absolute(path, db.system().current_directory());
roots.at(&absolute)
}
pub fn try_add_root(&self, db: &dyn Db, path: &SystemPath, kind: FileRootKind) -> FileRoot {
let mut roots = self.inner.roots.write().unwrap();
let absolute = SystemPath::absolute(path, db.system().current_directory());
roots.try_add(db, absolute, kind)
}
pub fn sync_all_recursive<P, I>(db: &mut dyn Db, paths: I)
where
P: AsRef<SystemPath>,
I: IntoIterator<Item = P>,
{
let current_directory = db.system().current_directory();
let paths = deduplicate_nested_paths(
paths
.into_iter()
.map(|path| SystemPath::absolute(path.as_ref(), current_directory)),
)
.collect::<BTreeSet<_>>();
if paths.is_empty() {
return;
}
let parents = paths
.iter()
.filter_map(|path| path.parent().map(SystemPath::to_path_buf))
.collect::<BTreeSet<_>>();
let inner = Arc::clone(&db.files().inner);
for entry in inner.system_by_path.iter_mut() {
let path = entry.key();
if paths
.range(..=path.to_path_buf())
.next_back()
.is_some_and(|candidate| path.starts_with(candidate.as_path()))
|| parents.contains(path)
{
File::sync_system_path(db, path, Some(*entry.value()));
}
}
}
pub fn sync_all(db: &mut dyn Db) {
tracing::debug!("Syncing all files");
let inner = Arc::clone(&db.files().inner);
for entry in inner.system_by_path.iter_mut() {
File::sync_system_path(db, entry.key(), Some(*entry.value()));
}
}
}
impl fmt::Debug for Files {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
if f.alternate() {
let mut map = f.debug_map();
for entry in self.inner.system_by_path.iter() {
map.entry(entry.key(), entry.value());
}
map.finish()
} else {
f.debug_struct("Files")
.field("system_by_path", &self.inner.system_by_path.len())
.field(
"system_virtual_by_path",
&self.inner.system_virtual_by_path.len(),
)
.field("vendored_by_path", &self.inner.vendored_by_path.len())
.finish()
}
}
}
impl std::panic::RefUnwindSafe for Files {}
#[salsa::input(heap_size=ruff_memory_usage::heap_size)]
#[derive(PartialOrd, Ord)]
pub struct File {
#[returns(ref)]
pub path: FilePath,
#[default]
#[returns(copy)]
pub permissions: Option<u32>,
#[default]
#[returns(copy)]
pub revision: FileRevision,
#[default]
#[returns(copy)]
pub status: FileStatus,
#[default]
#[returns(ref)]
pub source_text_override: Option<SourceText>,
}
impl get_size2::GetSize for File {}
struct SyncPathResult {
status_changed: bool,
}
impl File {
pub fn read_to_string(&self, db: &dyn Db) -> crate::system::Result<String> {
let path = self.path(db);
match path {
FilePath::System(system) => {
let _ = self.revision(db);
db.system().read_to_string(system)
}
FilePath::Vendored(vendored) => db.vendored().read_to_string(vendored),
FilePath::SystemVirtual(system_virtual) => {
let _ = self.revision(db);
db.system().read_virtual_path_to_string(system_virtual)
}
}
}
pub fn read_to_notebook(&self, db: &dyn Db) -> Result<Notebook, NotebookError> {
let path = self.path(db);
match path {
FilePath::System(system) => {
let _ = self.revision(db);
db.system().read_to_notebook(system)
}
FilePath::Vendored(_) => Err(NotebookError::Io(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"Reading a notebook from the vendored file system is not supported.",
))),
FilePath::SystemVirtual(system_virtual) => {
let _ = self.revision(db);
db.system().read_virtual_path_to_notebook(system_virtual)
}
}
}
pub fn sync_path(db: &mut dyn Db, path: &SystemPath) {
let absolute = SystemPath::absolute(path, db.system().current_directory());
let result = Self::sync_system_path(db, &absolute, None);
Self::touch_parent_directory_after_sync(db, &absolute, result);
}
pub fn sync_path_only(db: &mut dyn Db, path: &SystemPath) {
let absolute = SystemPath::absolute(path, db.system().current_directory());
Self::sync_system_path(db, &absolute, None);
}
pub fn sync_virtual_path(db: &mut dyn Db, path: &SystemVirtualPath) {
if let Some(virtual_file) = db.files().try_virtual_file(path) {
virtual_file.sync(db);
}
}
pub fn sync(self, db: &mut dyn Db) {
let path = self.path(db).clone();
match path {
FilePath::System(system) => {
let result = Self::sync_system_path(db, &system, Some(self));
Self::touch_parent_directory_after_sync(db, &system, result);
}
FilePath::Vendored(_) => {
}
FilePath::SystemVirtual(_) => {
VirtualFile(self).sync(db);
}
}
}
fn sync_system_path(db: &mut dyn Db, path: &SystemPath, file: Option<File>) -> SyncPathResult {
let Some(file) = file.or_else(|| db.files().try_system(db, path)) else {
return SyncPathResult {
status_changed: true,
};
};
let (status, revision, permission) = match db.system().path_metadata(path) {
Ok(metadata) if metadata.file_type().is_file() => (
FileStatus::Exists,
metadata.revision(),
metadata.permissions(),
),
Ok(metadata) if metadata.file_type().is_directory() => (
FileStatus::IsADirectory,
metadata.revision(),
metadata.permissions(),
),
_ => (FileStatus::NotFound, FileRevision::zero(), None),
};
let mut clear_override = false;
let old_status = file.status(db);
let status_changed = old_status != status;
if status_changed {
tracing::debug!("Updating the status of `{}`", file.path(db));
file.set_status(db).to(status);
clear_override = true;
}
if file.revision(db) != revision {
tracing::debug!("Updating the revision of `{}`", file.path(db));
file.set_revision(db).to(revision);
clear_override = true;
}
if file.permissions(db) != permission {
tracing::debug!("Updating the permissions of `{}`", file.path(db));
file.set_permissions(db).to(permission);
}
if clear_override && file.source_text_override(db).is_some() {
file.set_source_text_override(db).to(None);
}
SyncPathResult { status_changed }
}
fn touch_parent_directory_after_sync(
db: &mut dyn Db,
path: &SystemPath,
result: SyncPathResult,
) {
if result.status_changed
&& let Some(parent) = path.parent()
{
Self::sync_system_path(db, parent, None);
}
}
pub fn exists(self, db: &dyn Db) -> bool {
self.status(db) == FileStatus::Exists
}
pub fn is_stub(self, db: &dyn Db) -> bool {
self.source_type(db).is_stub()
}
pub fn is_package_stub(self, db: &dyn Db) -> bool {
self.path(db).as_str().ends_with("__init__.pyi")
}
pub fn is_package(self, db: &dyn Db) -> bool {
let path = self.path(db).as_str();
path.ends_with("__init__.pyi") || path.ends_with("__init__.py")
}
pub fn source_type(self, db: &dyn Db) -> PySourceType {
match self.path(db) {
FilePath::System(path) => path
.extension()
.map_or(PySourceType::Python, PySourceType::from_extension),
FilePath::Vendored(_) => PySourceType::Stub,
FilePath::SystemVirtual(path) => path
.extension()
.map_or(PySourceType::Python, PySourceType::from_extension),
}
}
}
impl fmt::Debug for File {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
salsa::with_attached_database(|db| {
if f.alternate() {
f.debug_struct("File")
.field("path", &self.path(db))
.field("status", &self.status(db))
.field("permissions", &self.permissions(db))
.field("revision", &self.revision(db))
.finish()
} else {
f.debug_tuple("File").field(&self.path(db)).finish()
}
})
.unwrap_or_else(|| f.debug_tuple("file").field(&self.as_id()).finish())
}
}
#[derive(Copy, Clone, Debug)]
pub struct VirtualFile(File);
impl VirtualFile {
pub fn file(&self) -> File {
self.0
}
pub fn sync(&self, db: &mut dyn Db) {
let file = self.0;
tracing::debug!("Updating the revision of `{}`", file.path(db));
let current_revision = file.revision(db);
file.set_revision(db)
.to(FileRevision::new(current_revision.as_u128() + 1));
}
pub fn close(&self, db: &mut dyn Db) {
tracing::debug!("Closing virtual file `{}`", self.0.path(db));
self.0.set_status(db).to(FileStatus::NotFound);
}
}
mod private {
#[derive(Copy, Clone, Debug, Eq, PartialEq, Default, get_size2::GetSize)]
pub enum FileStatus {
#[default]
Exists,
IsADirectory,
NotFound,
}
}
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub enum FileError {
IsADirectory,
NotFound,
}
impl fmt::Display for FileError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::fmt::Result {
match self {
FileError::IsADirectory => f.write_str("Is a directory"),
FileError::NotFound => f.write_str("Not found"),
}
}
}
impl std::error::Error for FileError {}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct FileRange {
file: File,
range: TextRange,
}
impl FileRange {
pub const fn new(file: File, range: TextRange) -> Self {
Self { file, range }
}
pub const fn file(&self) -> File {
self.file
}
}
impl Ranged for FileRange {
#[inline]
fn range(&self) -> TextRange {
self.range
}
}
impl TryFrom<&Span> for FileRange {
type Error = ();
fn try_from(value: &Span) -> Result<Self, Self::Error> {
let UnifiedFile::Ty(file) = value.file() else {
return Err(());
};
Ok(Self {
file: *file,
range: value.range().ok_or(())?,
})
}
}
impl TryFrom<Span> for FileRange {
type Error = ();
fn try_from(value: Span) -> Result<Self, Self::Error> {
Self::try_from(&value)
}
}
#[cfg(test)]
mod tests {
use salsa::Setter;
use crate::Db as _;
use crate::file_revision::FileRevision;
use crate::files::{File, FileError, system_path_to_file, vendored_path_to_file};
use crate::source::source_text;
use crate::system::{DbWithWritableSystem as _, SystemPath};
use crate::tests::TestDb;
use crate::vendored::VendoredFileSystemBuilder;
use zip::CompressionMethod;
#[test]
fn system_existing_file() -> crate::system::Result<()> {
let mut db = TestDb::new();
db.write_file("test.py", "print('Hello world')")?;
let test = system_path_to_file(&db, "test.py").expect("File to exist.");
assert_eq!(test.permissions(&db), Some(0o755));
assert_ne!(test.revision(&db), FileRevision::zero());
assert_eq!(&test.read_to_string(&db)?, "print('Hello world')");
Ok(())
}
#[test]
fn system_non_existing_file() {
let db = TestDb::new();
let test = system_path_to_file(&db, "test.py");
assert_eq!(test, Err(FileError::NotFound));
}
#[test]
fn system_normalize_paths() {
#[track_caller]
fn assert_normalized_path(db: &TestDb, path: &str, canonical: File) {
assert_eq!(system_path_to_file(db, path), Ok(canonical));
assert_eq!(
db.files().try_system(db, SystemPath::new(path)),
Some(canonical)
);
}
let mut db = TestDb::new();
db.write_file("/foo/bar.py", "x = 1").unwrap();
db.write_file("/foo/baz/bar.py", "x = 2").unwrap();
let canonical = system_path_to_file(&db, "/foo/bar.py").unwrap();
assert_normalized_path(&db, "foo/bar.py", canonical);
assert_normalized_path(&db, "/foo//bar.py", canonical);
assert_normalized_path(&db, "/foo/./bar.py", canonical);
assert_normalized_path(&db, "/foo/baz/../bar.py", canonical);
let distinct = system_path_to_file(&db, "/foo/baz/bar.py").unwrap();
assert_ne!(canonical, distinct);
}
#[test]
#[should_panic]
fn freeze_applies_to_new_file_overrides() {
let mut db = TestDb::new();
db.write_file("test.py", "x = 1").unwrap();
db.files().freeze();
let file = system_path_to_file(&db, "test.py").unwrap();
let source = source_text(&db, file);
file.set_source_text_override(&mut db).to(Some(source));
}
#[test]
fn freeze_does_not_change_existing_files() {
let mut db = TestDb::new();
db.write_file("test.py", "x = 1").unwrap();
let file = system_path_to_file(&db, "test.py").unwrap();
db.files().freeze();
let source = source_text(&db, file);
file.set_source_text_override(&mut db)
.to(Some(source.clone()));
assert_eq!(file.source_text_override(&db).as_ref(), Some(&source));
}
#[test]
fn stubbed_vendored_file() -> crate::system::Result<()> {
let mut db = TestDb::new();
let mut vendored_builder = VendoredFileSystemBuilder::new(CompressionMethod::Stored);
vendored_builder
.add_file("test.pyi", "def foo() -> str")
.unwrap();
let vendored = vendored_builder.finish().unwrap();
db.with_vendored(vendored);
let test = vendored_path_to_file(&db, "test.pyi").expect("Vendored file to exist.");
assert_eq!(test.permissions(&db), Some(0o444));
assert_ne!(test.revision(&db), FileRevision::zero());
assert_eq!(&test.read_to_string(&db)?, "def foo() -> str");
Ok(())
}
#[test]
fn stubbed_vendored_file_non_existing() {
let db = TestDb::new();
assert_eq!(
vendored_path_to_file(&db, "test.py"),
Err(FileError::NotFound)
);
}
}