mago_database/
lib.rs

1//! High-performance file database for PHP projects.
2//!
3//! This crate provides an efficient in-memory database for managing collections of PHP source files.
4//! It offers two complementary database types optimized for different access patterns:
5//!
6//! - [`Database`]: Mutable builder optimized for modifications (add, update, delete)
7//! - [`ReadDatabase`]: Immutable snapshot optimized for high-performance reads
8//!
9//! # Architecture
10//!
11//! The database uses a two-phase approach:
12//!
13//! 1. **Build Phase**: Use [`Database`] to load files, make modifications, and track changes
14//! 2. **Query Phase**: Convert to [`ReadDatabase`] via [`Database::read_only`] for fast lookups
15//!
16//! # Key Features
17//!
18//! - **Fast Lookups**: O(1) average-time access by ID, name, or filesystem path
19//! - **Change Tracking**: Record and batch apply file modifications via [`ChangeLog`]
20//! - **Deterministic Iteration**: [`ReadDatabase`] guarantees consistent iteration order
21//! - **Parallel Operations**: Concurrent file I/O and processing support
22//! - **Type Safety**: Strong typing with stable [`FileId`] handles
23//!
24//! # Common Workflow
25//!
26//! ## Loading Files
27//!
28//! Use [`loader::DatabaseLoader`] to scan a project directory:
29//!
30//! The loader handles file discovery, exclusion patterns, and parallel loading.
31//!
32//! ## Querying Files
33//!
34//! Both database types implement [`DatabaseReader`] for uniform access:
35//!
36//! ## Modifying Files
37//!
38//! Use [`ChangeLog`] to batch modifications:
39//!
40//! Changes can be applied to the database and optionally written to disk in parallel.
41//!
42//! # Performance Characteristics
43//!
44//! ## Database (Mutable)
45//!
46//! - Add/Update/Delete: O(1) average
47//! - Lookup by ID/name: O(1) average
48//! - Iteration: Unordered
49//! - Memory: ~2x file count (maps for bidirectional lookup)
50//!
51//! ## `ReadDatabase` (Immutable)
52//!
53//! - Creation: O(n log n) for sorting
54//! - Lookup by ID/name/path: O(1) average
55//! - Iteration: Deterministic, sorted by `FileId`
56//! - Memory: ~3x file count (vector + 3 index maps)
57//!
58//! # Thread Safety
59//!
60//! [`Database`] is not thread-safe and should be used from a single thread during construction.
61//! [`ReadDatabase`] can be freely shared across threads for concurrent read access.
62
63use std::borrow::Cow;
64use std::path::Path;
65use std::path::PathBuf;
66use std::sync::Arc;
67
68use foldhash::HashMap;
69use foldhash::HashMapExt;
70use rayon::iter::IntoParallelIterator;
71use rayon::iter::ParallelIterator;
72use serde::Deserialize;
73use serde::Serialize;
74
75use crate::change::Change;
76use crate::change::ChangeLog;
77use crate::error::DatabaseError;
78use crate::exclusion::Exclusion;
79use crate::file::File;
80use crate::file::FileId;
81use crate::file::FileType;
82use crate::file::line_starts;
83use crate::operation::FilesystemOperation;
84
85mod utils;
86
87pub mod change;
88pub mod error;
89pub mod exclusion;
90pub mod file;
91pub mod loader;
92pub mod watcher;
93
94mod operation;
95
96/// Configuration for database loading and watching.
97#[derive(Debug, Clone, Serialize, Deserialize)]
98pub struct DatabaseConfiguration<'a> {
99    pub workspace: Cow<'a, Path>,
100    /// Paths or glob patterns for source files.
101    /// Can be directory paths (e.g., "src") or glob patterns (e.g., "src/**/*.php")
102    pub paths: Vec<Cow<'a, str>>,
103    /// Paths or glob patterns for included files.
104    /// Can be directory paths (e.g., "vendor") or glob patterns (e.g., "vendor/**/*.php")
105    pub includes: Vec<Cow<'a, str>>,
106    pub excludes: Vec<Exclusion<'a>>,
107    pub extensions: Vec<Cow<'a, str>>,
108    /// Settings for glob pattern matching behavior.
109    pub glob: GlobSettings,
110}
111
112/// Settings for glob pattern matching behavior.
113///
114/// All defaults match the `globset` crate defaults for backwards compatibility.
115#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
116pub struct GlobSettings {
117    /// Match patterns case-insensitively.
118    ///
119    /// Default: `false`.
120    pub case_insensitive: bool,
121    /// When `true`, a single `*` does not match path separators (`/`).
122    /// This makes `src/*/Test` match only `src/foo/Test`, not `src/foo/bar/Test`.
123    /// Use `**` for recursive matching.
124    ///
125    /// Default: `false`.
126    pub literal_separator: bool,
127    /// Whether `\` escapes special characters in patterns.
128    ///
129    /// Default: `true`.
130    pub backslash_escape: bool,
131    /// Whether an empty case in alternates is allowed (e.g., `{,a}` matches `""` and `"a"`).
132    ///
133    /// Default: `false`.
134    pub empty_alternates: bool,
135}
136
137impl Default for GlobSettings {
138    fn default() -> Self {
139        Self {
140            case_insensitive: false,
141            literal_separator: false,
142            backslash_escape: !std::path::is_separator('\\'),
143            empty_alternates: false,
144        }
145    }
146}
147
148impl<'a> DatabaseConfiguration<'a> {
149    pub fn new(
150        workspace: &'a Path,
151        paths: Vec<&'a str>,
152        includes: Vec<&'a str>,
153        excludes: Vec<Exclusion<'a>>,
154        extensions: Vec<&'a str>,
155    ) -> Self {
156        let paths = paths.into_iter().map(Cow::Borrowed).collect();
157        let includes = includes.into_iter().map(Cow::Borrowed).collect();
158
159        let excludes = excludes
160            .into_iter()
161            .filter_map(|exclusion| match exclusion {
162                Exclusion::Path(p) => Some(if p.is_absolute() {
163                    Exclusion::Path(p)
164                } else {
165                    workspace.join(p).canonicalize().ok().map(Cow::Owned).map(Exclusion::Path)?
166                }),
167                Exclusion::Pattern(pat) => Some(Exclusion::Pattern(pat)),
168            })
169            .collect();
170
171        let extensions = extensions.into_iter().map(Cow::Borrowed).collect();
172
173        Self {
174            workspace: Cow::Borrowed(workspace),
175            paths,
176            includes,
177            excludes,
178            extensions,
179            glob: GlobSettings::default(),
180        }
181    }
182
183    #[must_use]
184    pub fn into_static(self) -> DatabaseConfiguration<'static> {
185        DatabaseConfiguration {
186            workspace: Cow::Owned(self.workspace.into_owned()),
187            paths: self.paths.into_iter().map(|s| Cow::Owned(s.into_owned())).collect(),
188            includes: self.includes.into_iter().map(|s| Cow::Owned(s.into_owned())).collect(),
189            excludes: self
190                .excludes
191                .into_iter()
192                .map(|e| match e {
193                    Exclusion::Path(p) => Exclusion::Path(Cow::Owned(p.into_owned())),
194                    Exclusion::Pattern(pat) => Exclusion::Pattern(Cow::Owned(pat.into_owned())),
195                })
196                .collect(),
197            extensions: self.extensions.into_iter().map(|s| Cow::Owned(s.into_owned())).collect(),
198            glob: self.glob,
199        }
200    }
201}
202
203/// Mutable database for managing project files with add/update/delete operations.
204#[derive(Debug, Clone, Serialize, Deserialize)]
205pub struct Database<'a> {
206    files: HashMap<Cow<'static, str>, Arc<File>>,
207    id_to_name: HashMap<FileId, Cow<'static, str>>,
208    pub(crate) configuration: DatabaseConfiguration<'a>,
209}
210
211/// Immutable, read-optimized snapshot of the database.
212#[derive(Debug)]
213pub struct ReadDatabase {
214    files: Vec<Arc<File>>,
215    id_to_index: HashMap<FileId, usize>,
216    name_to_index: HashMap<Cow<'static, str>, usize>,
217    path_to_index: HashMap<PathBuf, usize>,
218}
219
220impl<'a> Database<'a> {
221    #[must_use]
222    pub fn new(configuration: DatabaseConfiguration<'a>) -> Self {
223        Self { files: HashMap::default(), id_to_name: HashMap::default(), configuration }
224    }
225
226    #[must_use]
227    pub fn single(file: File, configuration: DatabaseConfiguration<'a>) -> Self {
228        let mut db = Self::new(configuration);
229        db.add(file);
230        db
231    }
232
233    /// Reserves capacity for at least `additional` more files.
234    pub fn reserve(&mut self, additional: usize) {
235        self.files.reserve(additional);
236        self.id_to_name.reserve(additional);
237    }
238
239    pub fn add(&mut self, file: File) -> FileId {
240        let name = file.name.clone();
241        let id = file.id;
242
243        if let Some(old_file) = self.files.insert(name.clone(), Arc::new(file)) {
244            self.id_to_name.remove(&old_file.id);
245        }
246
247        self.id_to_name.insert(id, name);
248
249        id
250    }
251
252    /// Updates a file's content using its stable `FileId`.
253    ///
254    /// This recalculates derived data like file size, line endings, and `FileRevision`.
255    /// If another `ReadDatabase` snapshot holds a reference to the file (preventing in-place
256    /// mutation), a new `Arc<File>` is created with the updated contents.
257    ///
258    /// Returns `true` if a file with the given ID was found and updated.
259    pub fn update(&mut self, id: FileId, new_contents: Cow<'static, str>) -> bool {
260        let Some(name) = self.id_to_name.get(&id) else {
261            return false;
262        };
263
264        let Some(arc) = self.files.get_mut(name) else {
265            return false;
266        };
267
268        if let Some(file) = Arc::get_mut(arc) {
269            file.contents = new_contents;
270            file.size = file.contents.len() as u32;
271            file.lines = line_starts(file.contents.as_ref());
272        } else {
273            // other Arc clones exist (e.g., from a ReadDatabase snapshot).
274            // Create a new File with updated contents and replace the Arc.
275            let old = &**arc;
276            *arc = Arc::new(File::new(old.name.clone(), old.file_type, old.path.clone(), new_contents));
277        }
278
279        true
280    }
281
282    /// Deletes a file from the database using its stable `FileId`.
283    ///
284    /// Returns `true` if a file with the given ID was found and removed.
285    pub fn delete(&mut self, id: FileId) -> bool {
286        if let Some(name) = self.id_to_name.remove(&id) { self.files.remove(&name).is_some() } else { false }
287    }
288
289    /// Commits a [`ChangeLog`], applying all its recorded operations to the database
290    /// and optionally writing them to the filesystem.
291    ///
292    /// # Arguments
293    ///
294    /// * `change_log`: The log of changes to apply.
295    /// * `write_to_disk`: If `true`, changes for files that have a filesystem
296    ///   path will be written to disk in parallel.
297    ///
298    /// # Errors
299    ///
300    /// Returns a [`DatabaseError`] if the log cannot be consumed or if any
301    /// filesystem operation fails.
302    pub fn commit(&mut self, change_log: ChangeLog, write_to_disk: bool) -> Result<(), DatabaseError> {
303        let changes = change_log.into_inner()?;
304        let mut fs_operations = if write_to_disk { Vec::new() } else { Vec::with_capacity(0) };
305
306        for change in changes {
307            match change {
308                Change::Add(file) => {
309                    if write_to_disk && let Some(path) = &file.path {
310                        fs_operations.push(FilesystemOperation::Write(path.clone(), file.contents.clone()));
311                    }
312
313                    self.add(file);
314                }
315                Change::Update(id, contents) => {
316                    if write_to_disk
317                        && let Ok(file) = self.get(&id)
318                        && let Some(path) = &file.path
319                    {
320                        fs_operations.push(FilesystemOperation::Write(path.clone(), contents.clone()));
321                    }
322
323                    self.update(id, contents);
324                }
325                Change::Delete(id) => {
326                    if write_to_disk
327                        && let Ok(file) = self.get(&id)
328                        && let Some(path) = &file.path
329                    {
330                        fs_operations.push(FilesystemOperation::Delete(path.clone()));
331                    }
332
333                    self.delete(id);
334                }
335            }
336        }
337
338        if write_to_disk {
339            fs_operations.into_par_iter().try_for_each(|op| -> Result<(), DatabaseError> { op.execute() })?;
340        }
341
342        Ok(())
343    }
344
345    /// Creates an independent, immutable snapshot of the database.
346    ///
347    /// This is a potentially expensive one-time operation as it **clones** all file
348    /// data. The resulting [`ReadDatabase`] is highly optimized for fast reads and
349    /// guarantees a deterministic iteration order. The original `Database` is not
350    /// consumed and can continue to be used.
351    #[must_use]
352    pub fn read_only(&self) -> ReadDatabase {
353        let mut files_vec: Vec<Arc<File>> = self.files.values().cloned().collect();
354        files_vec.sort_unstable_by_key(|f| f.id);
355
356        let mut id_to_index = HashMap::with_capacity(files_vec.len());
357        let mut name_to_index = HashMap::with_capacity(files_vec.len());
358        let mut path_to_index = HashMap::with_capacity(files_vec.len());
359
360        for (index, file) in files_vec.iter().enumerate() {
361            id_to_index.insert(file.id, index);
362            name_to_index.insert(file.name.clone(), index);
363            if let Some(path) = &file.path {
364                path_to_index.insert(path.clone(), index);
365            }
366        }
367
368        ReadDatabase { files: files_vec, id_to_index, name_to_index, path_to_index }
369    }
370}
371
372impl ReadDatabase {
373    #[must_use]
374    pub fn empty() -> Self {
375        Self {
376            files: Vec::with_capacity(0),
377            id_to_index: HashMap::with_capacity(0),
378            name_to_index: HashMap::with_capacity(0),
379            path_to_index: HashMap::with_capacity(0),
380        }
381    }
382
383    /// Creates a new `ReadDatabase` containing only a single file.
384    ///
385    /// This is a convenience constructor for situations, such as testing or
386    /// single-file tools, where an operation requires a [`DatabaseReader`]
387    /// implementation but only needs to be aware of one file.
388    ///
389    /// # Arguments
390    ///
391    /// * `file`: The single `File` to include in the database.
392    #[must_use]
393    pub fn single(file: File) -> Self {
394        let mut id_to_index = HashMap::with_capacity(1);
395        let mut name_to_index = HashMap::with_capacity(1);
396        let mut path_to_index = HashMap::with_capacity(1);
397
398        id_to_index.insert(file.id, 0);
399        name_to_index.insert(file.name.clone(), 0);
400        if let Some(path) = &file.path {
401            path_to_index.insert(path.clone(), 0);
402        }
403
404        Self { files: vec![Arc::new(file)], id_to_index, name_to_index, path_to_index }
405    }
406}
407
408/// A universal interface for reading data from any database implementation.
409///
410/// This trait provides a common API for querying file data, abstracting over
411/// whether the underlying source is the mutable [`Database`] or the read-optimized
412/// [`ReadDatabase`]. This allows for writing generic code that can operate on either.
413pub trait DatabaseReader {
414    /// Retrieves a file's stable ID using its logical name.
415    fn get_id(&self, name: &str) -> Option<FileId>;
416
417    /// Retrieves a reference to a file using its stable `FileId`.
418    ///
419    /// # Errors
420    ///
421    /// Returns `DatabaseError::FileNotFound` if no file with the given ID exists.
422    fn get(&self, id: &FileId) -> Result<Arc<File>, DatabaseError>;
423
424    /// Retrieves a reference to a file using its stable `FileId`.
425    ///
426    /// # Errors
427    ///
428    /// Returns `DatabaseError::FileNotFound` if no file with the given ID exists.
429    fn get_ref(&self, id: &FileId) -> Result<&File, DatabaseError>;
430
431    /// Retrieves a reference to a file using its logical name.
432    ///
433    /// # Errors
434    ///
435    /// Returns `DatabaseError::FileNotFound` if no file with the given name exists.
436    fn get_by_name(&self, name: &str) -> Result<Arc<File>, DatabaseError>;
437
438    /// Retrieves a reference to a file by its absolute filesystem path.
439    ///
440    /// # Errors
441    ///
442    /// Returns `DatabaseError::FileNotFound` if no file with the given path exists.
443    fn get_by_path(&self, path: &Path) -> Result<Arc<File>, DatabaseError>;
444
445    /// Returns an iterator over all files in the database.
446    ///
447    /// The order is not guaranteed for `Database`, but is sorted by `FileId`
448    /// for `ReadDatabase`, providing deterministic iteration.
449    fn files(&self) -> impl Iterator<Item = Arc<File>>;
450
451    /// Returns an iterator over all files of a specific `FileType`.
452    fn files_with_type(&self, file_type: FileType) -> impl Iterator<Item = Arc<File>> {
453        self.files().filter(move |file| file.file_type == file_type)
454    }
455
456    /// Returns an iterator over all files that do not match a specific `FileType`.
457    fn files_without_type(&self, file_type: FileType) -> impl Iterator<Item = Arc<File>> {
458        self.files().filter(move |file| file.file_type != file_type)
459    }
460
461    /// Returns an iterator over the stable IDs of all files in the database.
462    fn file_ids(&self) -> impl Iterator<Item = FileId> {
463        self.files().map(|file| file.id)
464    }
465
466    /// Returns an iterator over the stable IDs of all files of a specific `FileType`.
467    fn file_ids_with_type(&self, file_type: FileType) -> impl Iterator<Item = FileId> {
468        self.files_with_type(file_type).map(|file| file.id)
469    }
470
471    /// Returns an iterator over the stable IDs of all files that do not match a specific `FileType`.
472    fn file_ids_without_type(&self, file_type: FileType) -> impl Iterator<Item = FileId> {
473        self.files_without_type(file_type).map(|file| file.id)
474    }
475
476    /// Returns the total number of files in the database.
477    fn len(&self) -> usize;
478
479    /// Returns `true` if the database contains no files.
480    fn is_empty(&self) -> bool {
481        self.len() == 0
482    }
483}
484
485impl DatabaseReader for Database<'_> {
486    fn get_id(&self, name: &str) -> Option<FileId> {
487        self.files.get(name).map(|f| f.id)
488    }
489
490    fn get(&self, id: &FileId) -> Result<Arc<File>, DatabaseError> {
491        let name = self.id_to_name.get(id).ok_or(DatabaseError::FileNotFound)?;
492        let file = self.files.get(name).ok_or(DatabaseError::FileNotFound)?;
493
494        Ok(file.clone())
495    }
496
497    fn get_ref(&self, id: &FileId) -> Result<&File, DatabaseError> {
498        let name = self.id_to_name.get(id).ok_or(DatabaseError::FileNotFound)?;
499        self.files.get(name).map(std::convert::AsRef::as_ref).ok_or(DatabaseError::FileNotFound)
500    }
501
502    fn get_by_name(&self, name: &str) -> Result<Arc<File>, DatabaseError> {
503        self.files.get(name).cloned().ok_or(DatabaseError::FileNotFound)
504    }
505
506    fn get_by_path(&self, path: &Path) -> Result<Arc<File>, DatabaseError> {
507        self.files.values().find(|file| file.path.as_deref() == Some(path)).cloned().ok_or(DatabaseError::FileNotFound)
508    }
509
510    fn files(&self) -> impl Iterator<Item = Arc<File>> {
511        self.files.values().cloned()
512    }
513
514    fn len(&self) -> usize {
515        self.files.len()
516    }
517}
518
519impl DatabaseReader for ReadDatabase {
520    fn get_id(&self, name: &str) -> Option<FileId> {
521        self.name_to_index.get(name).and_then(|&i| self.files.get(i)).map(|f| f.id)
522    }
523
524    fn get(&self, id: &FileId) -> Result<Arc<File>, DatabaseError> {
525        let index = self.id_to_index.get(id).ok_or(DatabaseError::FileNotFound)?;
526
527        self.files.get(*index).cloned().ok_or(DatabaseError::FileNotFound)
528    }
529
530    fn get_ref(&self, id: &FileId) -> Result<&File, DatabaseError> {
531        let index = self.id_to_index.get(id).ok_or(DatabaseError::FileNotFound)?;
532
533        self.files.get(*index).map(std::convert::AsRef::as_ref).ok_or(DatabaseError::FileNotFound)
534    }
535
536    fn get_by_name(&self, name: &str) -> Result<Arc<File>, DatabaseError> {
537        self.name_to_index.get(name).and_then(|&i| self.files.get(i)).cloned().ok_or(DatabaseError::FileNotFound)
538    }
539
540    fn get_by_path(&self, path: &Path) -> Result<Arc<File>, DatabaseError> {
541        self.path_to_index.get(path).and_then(|&i| self.files.get(i)).cloned().ok_or(DatabaseError::FileNotFound)
542    }
543
544    fn files(&self) -> impl Iterator<Item = Arc<File>> {
545        self.files.iter().cloned()
546    }
547
548    fn len(&self) -> usize {
549        self.files.len()
550    }
551}
mago_database/lib.rs

mago_database/
lib.rs
