mago_database/
lib.rs

1//! High-performance file database for PHP projects.
2//!
3//! This crate provides an efficient in-memory database for managing collections of PHP source files.
4//! It offers two complementary database types optimized for different access patterns:
5//!
6//! - [`Database`]: Mutable builder optimized for modifications (add, update, delete)
7//! - [`ReadDatabase`]: Immutable snapshot optimized for high-performance reads
8//!
9//! # Architecture
10//!
11//! The database uses a two-phase approach:
12//!
13//! 1. **Build Phase**: Use [`Database`] to load files, make modifications, and track changes
14//! 2. **Query Phase**: Convert to [`ReadDatabase`] via [`Database::read_only`] for fast lookups
15//!
16//! # Key Features
17//!
18//! - **Fast Lookups**: O(1) average-time access by ID, name, or filesystem path
19//! - **Change Tracking**: Record and batch apply file modifications via [`ChangeLog`]
20//! - **Deterministic Iteration**: [`ReadDatabase`] guarantees consistent iteration order
21//! - **Parallel Operations**: Concurrent file I/O and processing support
22//! - **Type Safety**: Strong typing with stable [`FileId`] handles
23//!
24//! # Common Workflow
25//!
26//! ## Loading Files
27//!
28//! Use [`loader::DatabaseLoader`] to scan a project directory:
29//!
30//! The loader handles file discovery, exclusion patterns, and parallel loading.
31//!
32//! ## Querying Files
33//!
34//! Both database types implement [`DatabaseReader`] for uniform access:
35//!
36//! ## Modifying Files
37//!
38//! Use [`ChangeLog`] to batch modifications:
39//!
40//! Changes can be applied to the database and optionally written to disk in parallel.
41//!
42//! # Performance Characteristics
43//!
44//! ## Database (Mutable)
45//!
46//! - Add/Update/Delete: O(1) average
47//! - Lookup by ID/name: O(1) average
48//! - Iteration: Unordered
49//! - Memory: ~2x file count (maps for bidirectional lookup)
50//!
51//! ## `ReadDatabase` (Immutable)
52//!
53//! - Creation: O(n log n) for sorting
54//! - Lookup by ID/name/path: O(1) average
55//! - Iteration: Deterministic, sorted by `FileId`
56//! - Memory: ~3x file count (vector + 3 index maps)
57//!
58//! # Thread Safety
59//!
60//! [`Database`] is not thread-safe and should be used from a single thread during construction.
61//! [`ReadDatabase`] can be freely shared across threads for concurrent read access.
62
63use std::borrow::Cow;
64use std::path::Path;
65use std::path::PathBuf;
66use std::sync::Arc;
67
68use ahash::HashMap;
69use ahash::HashMapExt;
70use rayon::iter::IntoParallelIterator;
71use rayon::iter::ParallelIterator;
72use serde::Deserialize;
73use serde::Serialize;
74
75use crate::change::Change;
76use crate::change::ChangeLog;
77use crate::error::DatabaseError;
78use crate::exclusion::Exclusion;
79use crate::file::File;
80use crate::file::FileId;
81use crate::file::FileType;
82use crate::file::line_starts;
83use crate::operation::FilesystemOperation;
84
85mod utils;
86
87pub mod change;
88pub mod error;
89pub mod exclusion;
90pub mod file;
91pub mod loader;
92pub mod watcher;
93
94mod operation;
95
96/// Configuration for database loading and watching.
97#[derive(Debug, Clone, Serialize, Deserialize)]
98pub struct DatabaseConfiguration<'a> {
99    pub workspace: Cow<'a, Path>,
100    /// Paths or glob patterns for source files.
101    /// Can be directory paths (e.g., "src") or glob patterns (e.g., "src/**/*.php")
102    pub paths: Vec<Cow<'a, str>>,
103    /// Paths or glob patterns for included files.
104    /// Can be directory paths (e.g., "vendor") or glob patterns (e.g., "vendor/**/*.php")
105    pub includes: Vec<Cow<'a, str>>,
106    pub excludes: Vec<Exclusion<'a>>,
107    pub extensions: Vec<Cow<'a, str>>,
108}
109
110impl<'a> DatabaseConfiguration<'a> {
111    pub fn new(
112        workspace: &'a Path,
113        paths: Vec<&'a str>,
114        includes: Vec<&'a str>,
115        excludes: Vec<Exclusion<'a>>,
116        extensions: Vec<&'a str>,
117    ) -> Self {
118        let paths = paths.into_iter().map(Cow::Borrowed).collect();
119        let includes = includes.into_iter().map(Cow::Borrowed).collect();
120
121        let excludes = excludes
122            .into_iter()
123            .filter_map(|exclusion| match exclusion {
124                Exclusion::Path(p) => Some(if p.is_absolute() {
125                    Exclusion::Path(p)
126                } else {
127                    workspace.join(p).canonicalize().ok().map(Cow::Owned).map(Exclusion::Path)?
128                }),
129                Exclusion::Pattern(pat) => Some(Exclusion::Pattern(pat)),
130            })
131            .collect();
132
133        let extensions = extensions.into_iter().map(Cow::Borrowed).collect();
134
135        Self { workspace: Cow::Borrowed(workspace), paths, includes, excludes, extensions }
136    }
137
138    #[must_use]
139    pub fn into_static(self) -> DatabaseConfiguration<'static> {
140        DatabaseConfiguration {
141            workspace: Cow::Owned(self.workspace.into_owned()),
142            paths: self.paths.into_iter().map(|s| Cow::Owned(s.into_owned())).collect(),
143            includes: self.includes.into_iter().map(|s| Cow::Owned(s.into_owned())).collect(),
144            excludes: self
145                .excludes
146                .into_iter()
147                .map(|e| match e {
148                    Exclusion::Path(p) => Exclusion::Path(Cow::Owned(p.into_owned())),
149                    Exclusion::Pattern(pat) => Exclusion::Pattern(Cow::Owned(pat.into_owned())),
150                })
151                .collect(),
152            extensions: self.extensions.into_iter().map(|s| Cow::Owned(s.into_owned())).collect(),
153        }
154    }
155}
156
157/// Mutable database for managing project files with add/update/delete operations.
158#[derive(Debug, Clone, Serialize, Deserialize)]
159pub struct Database<'a> {
160    files: HashMap<Cow<'static, str>, Arc<File>>,
161    id_to_name: HashMap<FileId, Cow<'static, str>>,
162    pub(crate) configuration: DatabaseConfiguration<'a>,
163}
164
165/// Immutable, read-optimized snapshot of the database.
166#[derive(Debug)]
167pub struct ReadDatabase {
168    files: Vec<Arc<File>>,
169    id_to_index: HashMap<FileId, usize>,
170    name_to_index: HashMap<Cow<'static, str>, usize>,
171    path_to_index: HashMap<PathBuf, usize>,
172}
173
174impl<'a> Database<'a> {
175    #[must_use]
176    pub fn new(configuration: DatabaseConfiguration<'a>) -> Self {
177        Self { files: HashMap::default(), id_to_name: HashMap::default(), configuration }
178    }
179
180    #[must_use]
181    pub fn single(file: File, configuration: DatabaseConfiguration<'a>) -> Self {
182        let mut db = Self::new(configuration);
183        db.add(file);
184        db
185    }
186
187    pub fn add(&mut self, file: File) -> FileId {
188        let name = file.name.clone();
189        let id = file.id;
190
191        if let Some(old_file) = self.files.insert(name.clone(), Arc::new(file)) {
192            self.id_to_name.remove(&old_file.id);
193        }
194
195        self.id_to_name.insert(id, name);
196
197        id
198    }
199
200    /// Updates a file's content in-place using its stable `FileId`.
201    ///
202    /// This recalculates derived data like file size, line endings, and `FileRevision`.
203    /// Returns `true` if a file with the given ID was found and updated.
204    pub fn update(&mut self, id: FileId, new_contents: Cow<'static, str>) -> bool {
205        if let Some(name) = self.id_to_name.get(&id)
206            && let Some(file) = self.files.get_mut(name)
207            && let Some(file) = Arc::get_mut(file)
208        {
209            file.contents = new_contents;
210            file.size = file.contents.len() as u32;
211            file.lines = line_starts(file.contents.as_ref());
212            return true;
213        }
214        false
215    }
216
217    /// Deletes a file from the database using its stable `FileId`.
218    ///
219    /// Returns `true` if a file with the given ID was found and removed.
220    pub fn delete(&mut self, id: FileId) -> bool {
221        if let Some(name) = self.id_to_name.remove(&id) { self.files.remove(&name).is_some() } else { false }
222    }
223
224    /// Commits a [`ChangeLog`], applying all its recorded operations to the database
225    /// and optionally writing them to the filesystem.
226    ///
227    /// # Arguments
228    ///
229    /// * `change_log`: The log of changes to apply.
230    /// * `write_to_disk`: If `true`, changes for files that have a filesystem
231    ///   path will be written to disk in parallel.
232    ///
233    /// # Errors
234    ///
235    /// Returns a [`DatabaseError`] if the log cannot be consumed or if any
236    /// filesystem operation fails.
237    pub fn commit(&mut self, change_log: ChangeLog, write_to_disk: bool) -> Result<(), DatabaseError> {
238        let changes = change_log.into_inner()?;
239        let mut fs_operations = if write_to_disk { Vec::new() } else { Vec::with_capacity(0) };
240
241        for change in changes {
242            match change {
243                Change::Add(file) => {
244                    if write_to_disk && let Some(path) = &file.path {
245                        fs_operations.push(FilesystemOperation::Write(path.clone(), file.contents.clone()));
246                    }
247
248                    self.add(file);
249                }
250                Change::Update(id, contents) => {
251                    if write_to_disk
252                        && let Ok(file) = self.get(&id)
253                        && let Some(path) = &file.path
254                    {
255                        fs_operations.push(FilesystemOperation::Write(path.clone(), contents.clone()));
256                    }
257
258                    self.update(id, contents);
259                }
260                Change::Delete(id) => {
261                    if write_to_disk
262                        && let Ok(file) = self.get(&id)
263                        && let Some(path) = &file.path
264                    {
265                        fs_operations.push(FilesystemOperation::Delete(path.clone()));
266                    }
267
268                    self.delete(id);
269                }
270            }
271        }
272
273        if write_to_disk {
274            fs_operations.into_par_iter().try_for_each(|op| -> Result<(), DatabaseError> { op.execute() })?;
275        }
276
277        Ok(())
278    }
279
280    /// Creates an independent, immutable snapshot of the database.
281    ///
282    /// This is a potentially expensive one-time operation as it **clones** all file
283    /// data. The resulting [`ReadDatabase`] is highly optimized for fast reads and
284    /// guarantees a deterministic iteration order. The original `Database` is not
285    /// consumed and can continue to be used.
286    #[must_use]
287    pub fn read_only(&self) -> ReadDatabase {
288        let mut files_vec: Vec<Arc<File>> = self.files.values().cloned().collect();
289        files_vec.sort_unstable_by_key(|f| f.id);
290
291        let mut id_to_index = HashMap::with_capacity(files_vec.len());
292        let mut name_to_index = HashMap::with_capacity(files_vec.len());
293        let mut path_to_index = HashMap::with_capacity(files_vec.len());
294
295        for (index, file) in files_vec.iter().enumerate() {
296            id_to_index.insert(file.id, index);
297            name_to_index.insert(file.name.clone(), index);
298            if let Some(path) = &file.path {
299                path_to_index.insert(path.clone(), index);
300            }
301        }
302
303        ReadDatabase { files: files_vec, id_to_index, name_to_index, path_to_index }
304    }
305}
306
307impl ReadDatabase {
308    #[must_use]
309    pub fn empty() -> Self {
310        Self {
311            files: Vec::with_capacity(0),
312            id_to_index: HashMap::with_capacity(0),
313            name_to_index: HashMap::with_capacity(0),
314            path_to_index: HashMap::with_capacity(0),
315        }
316    }
317
318    /// Creates a new `ReadDatabase` containing only a single file.
319    ///
320    /// This is a convenience constructor for situations, such as testing or
321    /// single-file tools, where an operation requires a [`DatabaseReader`]
322    /// implementation but only needs to be aware of one file.
323    ///
324    /// # Arguments
325    ///
326    /// * `file`: The single `File` to include in the database.
327    #[must_use]
328    pub fn single(file: File) -> Self {
329        let mut id_to_index = HashMap::with_capacity(1);
330        let mut name_to_index = HashMap::with_capacity(1);
331        let mut path_to_index = HashMap::with_capacity(1);
332
333        id_to_index.insert(file.id, 0);
334        name_to_index.insert(file.name.clone(), 0);
335        if let Some(path) = &file.path {
336            path_to_index.insert(path.clone(), 0);
337        }
338
339        Self { files: vec![Arc::new(file)], id_to_index, name_to_index, path_to_index }
340    }
341}
342
343/// A universal interface for reading data from any database implementation.
344///
345/// This trait provides a common API for querying file data, abstracting over
346/// whether the underlying source is the mutable [`Database`] or the read-optimized
347/// [`ReadDatabase`]. This allows for writing generic code that can operate on either.
348pub trait DatabaseReader {
349    /// Retrieves a file's stable ID using its logical name.
350    fn get_id(&self, name: &str) -> Option<FileId>;
351
352    /// Retrieves a reference to a file using its stable `FileId`.
353    ///
354    /// # Errors
355    ///
356    /// Returns `DatabaseError::FileNotFound` if no file with the given ID exists.
357    fn get(&self, id: &FileId) -> Result<Arc<File>, DatabaseError>;
358
359    /// Retrieves a reference to a file using its stable `FileId`.
360    ///
361    /// # Errors
362    ///
363    /// Returns `DatabaseError::FileNotFound` if no file with the given ID exists.
364    fn get_ref(&self, id: &FileId) -> Result<&File, DatabaseError>;
365
366    /// Retrieves a reference to a file using its logical name.
367    ///
368    /// # Errors
369    ///
370    /// Returns `DatabaseError::FileNotFound` if no file with the given name exists.
371    fn get_by_name(&self, name: &str) -> Result<Arc<File>, DatabaseError>;
372
373    /// Retrieves a reference to a file by its absolute filesystem path.
374    ///
375    /// # Errors
376    ///
377    /// Returns `DatabaseError::FileNotFound` if no file with the given path exists.
378    fn get_by_path(&self, path: &Path) -> Result<Arc<File>, DatabaseError>;
379
380    /// Returns an iterator over all files in the database.
381    ///
382    /// The order is not guaranteed for `Database`, but is sorted by `FileId`
383    /// for `ReadDatabase`, providing deterministic iteration.
384    fn files(&self) -> impl Iterator<Item = Arc<File>>;
385
386    /// Returns an iterator over all files of a specific `FileType`.
387    fn files_with_type(&self, file_type: FileType) -> impl Iterator<Item = Arc<File>> {
388        self.files().filter(move |file| file.file_type == file_type)
389    }
390
391    /// Returns an iterator over all files that do not match a specific `FileType`.
392    fn files_without_type(&self, file_type: FileType) -> impl Iterator<Item = Arc<File>> {
393        self.files().filter(move |file| file.file_type != file_type)
394    }
395
396    /// Returns an iterator over the stable IDs of all files in the database.
397    fn file_ids(&self) -> impl Iterator<Item = FileId> {
398        self.files().map(|file| file.id)
399    }
400
401    /// Returns an iterator over the stable IDs of all files of a specific `FileType`.
402    fn file_ids_with_type(&self, file_type: FileType) -> impl Iterator<Item = FileId> {
403        self.files_with_type(file_type).map(|file| file.id)
404    }
405
406    /// Returns an iterator over the stable IDs of all files that do not match a specific `FileType`.
407    fn file_ids_without_type(&self, file_type: FileType) -> impl Iterator<Item = FileId> {
408        self.files_without_type(file_type).map(|file| file.id)
409    }
410
411    /// Returns the total number of files in the database.
412    fn len(&self) -> usize;
413
414    /// Returns `true` if the database contains no files.
415    fn is_empty(&self) -> bool {
416        self.len() == 0
417    }
418}
419
420impl DatabaseReader for Database<'_> {
421    fn get_id(&self, name: &str) -> Option<FileId> {
422        self.files.get(name).map(|f| f.id)
423    }
424
425    fn get(&self, id: &FileId) -> Result<Arc<File>, DatabaseError> {
426        let name = self.id_to_name.get(id).ok_or(DatabaseError::FileNotFound)?;
427        let file = self.files.get(name).ok_or(DatabaseError::FileNotFound)?;
428
429        Ok(file.clone())
430    }
431
432    fn get_ref(&self, id: &FileId) -> Result<&File, DatabaseError> {
433        let name = self.id_to_name.get(id).ok_or(DatabaseError::FileNotFound)?;
434        self.files.get(name).map(std::convert::AsRef::as_ref).ok_or(DatabaseError::FileNotFound)
435    }
436
437    fn get_by_name(&self, name: &str) -> Result<Arc<File>, DatabaseError> {
438        self.files.get(name).cloned().ok_or(DatabaseError::FileNotFound)
439    }
440
441    fn get_by_path(&self, path: &Path) -> Result<Arc<File>, DatabaseError> {
442        self.files.values().find(|file| file.path.as_deref() == Some(path)).cloned().ok_or(DatabaseError::FileNotFound)
443    }
444
445    fn files(&self) -> impl Iterator<Item = Arc<File>> {
446        self.files.values().cloned()
447    }
448
449    fn len(&self) -> usize {
450        self.files.len()
451    }
452}
453
454impl DatabaseReader for ReadDatabase {
455    fn get_id(&self, name: &str) -> Option<FileId> {
456        self.name_to_index.get(name).and_then(|&i| self.files.get(i)).map(|f| f.id)
457    }
458
459    fn get(&self, id: &FileId) -> Result<Arc<File>, DatabaseError> {
460        let index = self.id_to_index.get(id).ok_or(DatabaseError::FileNotFound)?;
461
462        self.files.get(*index).cloned().ok_or(DatabaseError::FileNotFound)
463    }
464
465    fn get_ref(&self, id: &FileId) -> Result<&File, DatabaseError> {
466        let index = self.id_to_index.get(id).ok_or(DatabaseError::FileNotFound)?;
467
468        self.files.get(*index).map(std::convert::AsRef::as_ref).ok_or(DatabaseError::FileNotFound)
469    }
470
471    fn get_by_name(&self, name: &str) -> Result<Arc<File>, DatabaseError> {
472        self.name_to_index.get(name).and_then(|&i| self.files.get(i)).cloned().ok_or(DatabaseError::FileNotFound)
473    }
474
475    fn get_by_path(&self, path: &Path) -> Result<Arc<File>, DatabaseError> {
476        self.path_to_index.get(path).and_then(|&i| self.files.get(i)).cloned().ok_or(DatabaseError::FileNotFound)
477    }
478
479    fn files(&self) -> impl Iterator<Item = Arc<File>> {
480        self.files.iter().cloned()
481    }
482
483    fn len(&self) -> usize {
484        self.files.len()
485    }
486}
mago_database/lib.rs

mago_database/
lib.rs
