mago_database/
lib.rs

1#![allow(clippy::exhaustive_enums)]
2
3//! High-performance file database for PHP projects.
4//!
5//! This crate provides an efficient in-memory database for managing collections of PHP source files.
6//! It offers two complementary database types optimized for different access patterns:
7//!
8//! - [`Database`]: Mutable builder optimized for modifications (add, update, delete)
9//! - [`ReadDatabase`]: Immutable snapshot optimized for high-performance reads
10//!
11//! # Architecture
12//!
13//! The database uses a two-phase approach:
14//!
15//! 1. **Build Phase**: Use [`Database`] to load files, make modifications, and track changes
16//! 2. **Query Phase**: Convert to [`ReadDatabase`] via [`Database::read_only`] for fast lookups
17//!
18//! # Key Features
19//!
20//! - **Fast Lookups**: O(1) average-time access by ID, name, or filesystem path
21//! - **Change Tracking**: Record and batch apply file modifications via [`ChangeLog`]
22//! - **Deterministic Iteration**: [`ReadDatabase`] guarantees consistent iteration order
23//! - **Parallel Operations**: Concurrent file I/O and processing support
24//! - **Type Safety**: Strong typing with stable [`FileId`] handles
25//!
26//! # Common Workflow
27//!
28//! ## Loading Files
29//!
30//! Use [`loader::DatabaseLoader`] to scan a project directory:
31//!
32//! The loader handles file discovery, exclusion patterns, and parallel loading.
33//!
34//! ## Querying Files
35//!
36//! Both database types implement [`DatabaseReader`] for uniform access:
37//!
38//! ## Modifying Files
39//!
40//! Use [`ChangeLog`] to batch modifications:
41//!
42//! Changes can be applied to the database and optionally written to disk in parallel.
43//!
44//! # Performance Characteristics
45//!
46//! ## Database (Mutable)
47//!
48//! - Add/Update/Delete: O(1) average
49//! - Lookup by ID/name: O(1) average
50//! - Iteration: Unordered
51//! - Memory: ~2x file count (maps for bidirectional lookup)
52//!
53//! ## `ReadDatabase` (Immutable)
54//!
55//! - Creation: O(n log n) for sorting
56//! - Lookup by ID/name/path: O(1) average
57//! - Iteration: Deterministic, sorted by `FileId`
58//! - Memory: ~3x file count (vector + 3 index maps)
59//!
60//! # Thread Safety
61//!
62//! [`Database`] is not thread-safe and should be used from a single thread during construction.
63//! [`ReadDatabase`] can be freely shared across threads for concurrent read access.
64
65use std::borrow::Cow;
66use std::path::Path;
67use std::path::PathBuf;
68use std::sync::Arc;
69
70use foldhash::HashMap;
71use foldhash::HashMapExt;
72use rayon::iter::IntoParallelIterator;
73use rayon::iter::ParallelIterator;
74
75use crate::change::Change;
76use crate::change::ChangeLog;
77use crate::error::DatabaseError;
78use crate::exclusion::Exclusion;
79use crate::file::File;
80use crate::file::FileId;
81use crate::file::FileType;
82use crate::file::line_starts;
83use crate::operation::FilesystemOperation;
84
85mod utils;
86
87pub mod change;
88pub mod error;
89pub mod exclusion;
90pub mod file;
91pub mod loader;
92pub mod matcher;
93pub mod membership;
94pub mod watcher;
95
96mod operation;
97
98/// Configuration for database loading and watching.
99#[derive(Debug, Clone)]
100#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
101pub struct DatabaseConfiguration<'config> {
102    pub workspace: Cow<'config, Path>,
103    /// Paths or glob patterns for source files.
104    /// Can be directory paths (e.g., "src") or glob patterns (e.g., "src/**/*.php")
105    pub paths: Vec<Cow<'config, [u8]>>,
106    /// Paths or glob patterns for included files.
107    /// Can be directory paths (e.g., "vendor") or glob patterns (e.g., "vendor/**/*.php")
108    pub includes: Vec<Cow<'config, [u8]>>,
109    pub patches: Vec<Cow<'config, [u8]>>,
110    pub excludes: Vec<Exclusion<'config>>,
111    pub extensions: Vec<Cow<'config, [u8]>>,
112    /// Settings for glob pattern matching behavior.
113    pub glob: GlobSettings,
114}
115
116/// Settings for glob pattern matching behavior.
117///
118/// All defaults match the `globset` crate defaults for backwards compatibility.
119#[derive(Debug, Clone, Copy)]
120#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
121pub struct GlobSettings {
122    /// Match patterns case-insensitively.
123    ///
124    /// Default: `false`.
125    pub case_insensitive: bool,
126    /// When `true`, a single `*` does not match path separators (`/`).
127    /// This makes `src/*/Test` match only `src/foo/Test`, not `src/foo/bar/Test`.
128    /// Use `**` for recursive matching.
129    ///
130    /// Default: `false`.
131    pub literal_separator: bool,
132    /// Whether `\` escapes special characters in patterns.
133    ///
134    /// Default: `true`.
135    pub backslash_escape: bool,
136    /// Whether an empty case in alternates is allowed (e.g., `{,a}` matches `""` and `"a"`).
137    ///
138    /// Default: `false`.
139    pub empty_alternates: bool,
140}
141
142impl Default for GlobSettings {
143    #[inline]
144    fn default() -> Self {
145        Self {
146            case_insensitive: false,
147            literal_separator: false,
148            backslash_escape: !std::path::is_separator('\\'),
149            empty_alternates: false,
150        }
151    }
152}
153
154impl<'config> DatabaseConfiguration<'config> {
155    #[inline]
156    #[must_use]
157    pub fn new(
158        workspace: &'config Path,
159        paths: Vec<&'config [u8]>,
160        includes: Vec<&'config [u8]>,
161        excludes: Vec<Exclusion<'config>>,
162        extensions: Vec<&'config [u8]>,
163    ) -> Self {
164        let paths = paths.into_iter().map(Cow::Borrowed).collect();
165        let includes = includes.into_iter().map(Cow::Borrowed).collect();
166
167        let excludes = excludes
168            .into_iter()
169            .filter_map(|exclusion| match exclusion {
170                Exclusion::Path(p) => Some(if p.is_absolute() {
171                    Exclusion::Path(p)
172                } else {
173                    workspace.join(p).canonicalize().ok().map(Cow::Owned).map(Exclusion::Path)?
174                }),
175                Exclusion::Pattern(pat) => Some(Exclusion::Pattern(pat)),
176            })
177            .collect();
178
179        let extensions = extensions.into_iter().map(Cow::Borrowed).collect();
180
181        Self {
182            workspace: Cow::Borrowed(workspace),
183            paths,
184            includes,
185            patches: Vec::new(),
186            excludes,
187            extensions,
188            glob: GlobSettings::default(),
189        }
190    }
191
192    #[inline]
193    #[must_use]
194    pub fn into_static(self) -> DatabaseConfiguration<'static> {
195        DatabaseConfiguration {
196            workspace: Cow::Owned(self.workspace.into_owned()),
197            paths: self.paths.into_iter().map(|s| Cow::Owned(s.into_owned())).collect(),
198            includes: self.includes.into_iter().map(|s| Cow::Owned(s.into_owned())).collect(),
199            patches: self.patches.into_iter().map(|s| Cow::Owned(s.into_owned())).collect(),
200            excludes: self
201                .excludes
202                .into_iter()
203                .map(|e| match e {
204                    Exclusion::Path(p) => Exclusion::Path(Cow::Owned(p.into_owned())),
205                    Exclusion::Pattern(pat) => Exclusion::Pattern(Cow::Owned(pat.into_owned())),
206                })
207                .collect(),
208            extensions: self.extensions.into_iter().map(|s| Cow::Owned(s.into_owned())).collect(),
209            glob: self.glob,
210        }
211    }
212}
213
214/// Mutable database for managing project files with add/update/delete operations.
215#[derive(Debug, Clone)]
216#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
217#[allow(clippy::field_scoped_visibility_modifiers)]
218pub struct Database<'config> {
219    files: HashMap<Cow<'static, [u8]>, Arc<File>>,
220    id_to_name: HashMap<FileId, Cow<'static, [u8]>>,
221    pub(crate) configuration: DatabaseConfiguration<'config>,
222}
223
224/// Immutable, read-optimized snapshot of the database.
225#[derive(Debug)]
226pub struct ReadDatabase {
227    files: Vec<Arc<File>>,
228    id_to_index: HashMap<FileId, usize>,
229    name_to_index: HashMap<Cow<'static, [u8]>, usize>,
230    path_to_index: HashMap<PathBuf, usize>,
231}
232
233impl<'config> Database<'config> {
234    #[inline]
235    #[must_use]
236    pub fn new(configuration: DatabaseConfiguration<'config>) -> Self {
237        Self { files: HashMap::default(), id_to_name: HashMap::default(), configuration }
238    }
239
240    #[inline]
241    #[must_use]
242    pub fn single(file: File, configuration: DatabaseConfiguration<'config>) -> Self {
243        let mut db = Self::new(configuration);
244        db.add(file);
245        db
246    }
247
248    /// Reserves capacity for at least `additional` more files.
249    #[inline]
250    pub fn reserve(&mut self, additional: usize) {
251        self.files.reserve(additional);
252        self.id_to_name.reserve(additional);
253    }
254
255    /// Adds files from a base database without replacing files already present in this database.
256    ///
257    /// This is useful when project files and a built-in prelude are loaded independently: project
258    /// files retain precedence while missing built-in files are moved into the project database.
259    #[inline]
260    pub fn merge_base(&mut self, base: Database<'_>) {
261        let Database { files, mut id_to_name, configuration: _ } = base;
262        self.reserve(files.len());
263
264        for (name, file) in files {
265            if self.files.contains_key(name.as_ref()) {
266                continue;
267            }
268
269            let id = file.id;
270            let id_name = id_to_name.remove(&id).unwrap_or_else(|| name.clone());
271            self.files.insert(name, file);
272            self.id_to_name.insert(id, id_name);
273        }
274    }
275
276    #[inline]
277    pub fn add(&mut self, file: File) -> FileId {
278        let name = file.name.clone();
279        let id = file.id;
280
281        if let Some(old_file) = self.files.insert(name.clone(), Arc::new(file)) {
282            self.id_to_name.remove(&old_file.id);
283        }
284
285        self.id_to_name.insert(id, name);
286
287        id
288    }
289
290    /// Updates a file's content using its stable `FileId`.
291    ///
292    /// This recalculates derived data like file size, line endings, and `FileRevision`.
293    /// If another `ReadDatabase` snapshot holds a reference to the file (preventing in-place
294    /// mutation), a new `Arc<File>` is created with the updated contents.
295    ///
296    /// Returns `true` if a file with the given ID was found and updated.
297    #[inline]
298    pub fn update(&mut self, id: FileId, new_contents: Cow<'static, [u8]>) -> bool {
299        let Some(name) = self.id_to_name.get(&id) else {
300            return false;
301        };
302
303        let Some(arc) = self.files.get_mut(name) else {
304            return false;
305        };
306
307        if let Some(file) = Arc::get_mut(arc) {
308            file.contents = new_contents;
309            file.size = file.contents.len() as u32;
310            file.lines = line_starts(file.contents.as_ref());
311        } else {
312            // other Arc clones exist (e.g., from a ReadDatabase snapshot).
313            // Create a new File with updated contents and replace the Arc.
314            let old = &**arc;
315            *arc = Arc::new(File::new(old.name.clone(), old.file_type, old.path.clone(), new_contents));
316        }
317
318        true
319    }
320
321    /// Deletes a file from the database using its stable `FileId`.
322    ///
323    /// Returns `true` if a file with the given ID was found and removed.
324    #[inline]
325    pub fn delete(&mut self, id: FileId) -> bool {
326        if let Some(name) = self.id_to_name.remove(&id) { self.files.remove(&name).is_some() } else { false }
327    }
328
329    /// Commits a [`ChangeLog`], applying all its recorded operations to the database
330    /// and optionally writing them to the filesystem.
331    ///
332    /// # Arguments
333    ///
334    /// * `change_log`: The log of changes to apply.
335    /// * `write_to_disk`: If `true`, changes for files that have a filesystem
336    ///   path will be written to disk in parallel.
337    ///
338    /// # Errors
339    ///
340    /// Returns a [`DatabaseError`] if the log cannot be consumed or if any
341    /// filesystem operation fails.
342    #[inline]
343    pub fn commit(&mut self, change_log: ChangeLog, write_to_disk: bool) -> Result<(), DatabaseError> {
344        let changes = change_log.into_inner()?;
345        let mut fs_operations = Vec::new();
346
347        for change in changes {
348            match change {
349                Change::Add(file) => {
350                    if write_to_disk && let Some(path) = &file.path {
351                        fs_operations.push(FilesystemOperation::Write(path.clone(), file.contents.clone()));
352                    }
353
354                    self.add(file);
355                }
356                Change::Update(id, contents) => {
357                    if write_to_disk
358                        && let Ok(file) = self.get(&id)
359                        && let Some(path) = &file.path
360                    {
361                        fs_operations.push(FilesystemOperation::Write(path.clone(), contents.clone()));
362                    }
363
364                    self.update(id, contents);
365                }
366                Change::Delete(id) => {
367                    if write_to_disk
368                        && let Ok(file) = self.get(&id)
369                        && let Some(path) = &file.path
370                    {
371                        fs_operations.push(FilesystemOperation::Delete(path.clone()));
372                    }
373
374                    self.delete(id);
375                }
376            }
377        }
378
379        if write_to_disk {
380            fs_operations.into_par_iter().try_for_each(|op| -> Result<(), DatabaseError> { op.execute() })?;
381        }
382
383        Ok(())
384    }
385
386    /// Creates an independent, immutable snapshot of the database.
387    ///
388    /// This is a potentially expensive one-time operation as it **clones** all file
389    /// data. The resulting [`ReadDatabase`] is highly optimized for fast reads and
390    /// guarantees a deterministic iteration order. The original `Database` is not
391    /// consumed and can continue to be used.
392    #[inline]
393    #[must_use]
394    pub fn read_only(&self) -> ReadDatabase {
395        let mut files_vec: Vec<Arc<File>> = self.files.values().cloned().collect();
396        files_vec.sort_unstable_by_key(|f| f.id);
397
398        let mut id_to_index = HashMap::with_capacity(files_vec.len());
399        let mut name_to_index = HashMap::with_capacity(files_vec.len());
400        let mut path_to_index = HashMap::with_capacity(files_vec.len());
401
402        for (index, file) in files_vec.iter().enumerate() {
403            id_to_index.insert(file.id, index);
404            name_to_index.insert(file.name.clone(), index);
405            if let Some(path) = &file.path {
406                path_to_index.insert(path.clone(), index);
407            }
408        }
409
410        ReadDatabase { files: files_vec, id_to_index, name_to_index, path_to_index }
411    }
412}
413
414impl ReadDatabase {
415    #[inline]
416    #[must_use]
417    pub fn empty() -> Self {
418        Self {
419            files: Vec::new(),
420            id_to_index: HashMap::new(),
421            name_to_index: HashMap::new(),
422            path_to_index: HashMap::new(),
423        }
424    }
425
426    /// Creates a new `ReadDatabase` containing only a single file.
427    ///
428    /// This is a convenience constructor for situations, such as testing or
429    /// single-file tools, where an operation requires a [`DatabaseReader`]
430    /// implementation but only needs to be aware of one file.
431    ///
432    /// # Arguments
433    ///
434    /// * `file`: The single `File` to include in the database.
435    #[inline]
436    #[must_use]
437    pub fn single(file: File) -> Self {
438        let mut id_to_index = HashMap::with_capacity(1);
439        let mut name_to_index = HashMap::with_capacity(1);
440        let mut path_to_index = HashMap::with_capacity(1);
441
442        id_to_index.insert(file.id, 0);
443        name_to_index.insert(file.name.clone(), 0);
444        if let Some(path) = &file.path {
445            path_to_index.insert(path.clone(), 0);
446        }
447
448        Self { files: vec![Arc::new(file)], id_to_index, name_to_index, path_to_index }
449    }
450}
451
452/// A universal interface for reading data from any database implementation.
453///
454/// This trait provides a common API for querying file data, abstracting over
455/// whether the underlying source is the mutable [`Database`] or the read-optimized
456/// [`ReadDatabase`]. This allows for writing generic code that can operate on either.
457pub trait DatabaseReader {
458    /// Retrieves a file's stable ID using its logical name.
459    fn get_id(&self, name: &[u8]) -> Option<FileId>;
460
461    /// Retrieves a reference to a file using its stable `FileId`.
462    ///
463    /// # Errors
464    ///
465    /// Returns `DatabaseError::FileNotFound` if no file with the given ID exists.
466    fn get(&self, id: &FileId) -> Result<Arc<File>, DatabaseError>;
467
468    /// Retrieves a reference to a file using its stable `FileId`.
469    ///
470    /// # Errors
471    ///
472    /// Returns `DatabaseError::FileNotFound` if no file with the given ID exists.
473    fn get_ref(&self, id: &FileId) -> Result<&File, DatabaseError>;
474
475    /// Retrieves a reference to a file using its logical name.
476    ///
477    /// # Errors
478    ///
479    /// Returns `DatabaseError::FileNotFound` if no file with the given name exists.
480    fn get_by_name(&self, name: &[u8]) -> Result<Arc<File>, DatabaseError>;
481
482    /// Retrieves a reference to a file by its absolute filesystem path.
483    ///
484    /// # Errors
485    ///
486    /// Returns `DatabaseError::FileNotFound` if no file with the given path exists.
487    fn get_by_path(&self, path: &Path) -> Result<Arc<File>, DatabaseError>;
488
489    /// Returns an iterator over all files in the database.
490    ///
491    /// The order is not guaranteed for `Database`, but is sorted by `FileId`
492    /// for `ReadDatabase`, providing deterministic iteration.
493    fn files(&self) -> impl Iterator<Item = Arc<File>>;
494
495    /// Returns an iterator over all files of a specific `FileType`.
496    #[inline]
497    fn files_with_type(&self, file_type: FileType) -> impl Iterator<Item = Arc<File>> {
498        self.files().filter(move |file| file.file_type == file_type)
499    }
500
501    /// Returns an iterator over all files that do not match a specific `FileType`.
502    #[inline]
503    fn files_without_type(&self, file_type: FileType) -> impl Iterator<Item = Arc<File>> {
504        self.files().filter(move |file| file.file_type != file_type)
505    }
506
507    /// Returns an iterator over the stable IDs of all files in the database.
508    #[inline]
509    fn file_ids(&self) -> impl Iterator<Item = FileId> {
510        self.files().map(|file| file.id)
511    }
512
513    /// Returns an iterator over the stable IDs of all files of a specific `FileType`.
514    #[inline]
515    fn file_ids_with_type(&self, file_type: FileType) -> impl Iterator<Item = FileId> {
516        self.files_with_type(file_type).map(|file| file.id)
517    }
518
519    /// Returns an iterator over the stable IDs of all files that do not match a specific `FileType`.
520    #[inline]
521    fn file_ids_without_type(&self, file_type: FileType) -> impl Iterator<Item = FileId> {
522        self.files_without_type(file_type).map(|file| file.id)
523    }
524
525    /// Returns the total number of files in the database.
526    fn len(&self) -> usize;
527
528    /// Returns `true` if the database contains no files.
529    #[inline]
530    fn is_empty(&self) -> bool {
531        self.len() == 0
532    }
533}
534
535impl DatabaseReader for Database<'_> {
536    #[inline]
537    fn get_id(&self, name: &[u8]) -> Option<FileId> {
538        self.files.get(name).map(|f| f.id)
539    }
540
541    #[inline]
542    fn get(&self, id: &FileId) -> Result<Arc<File>, DatabaseError> {
543        let name = self.id_to_name.get(id).ok_or(DatabaseError::FileNotFound)?;
544        let file = self.files.get(name).ok_or(DatabaseError::FileNotFound)?;
545
546        Ok(Arc::clone(file))
547    }
548
549    #[inline]
550    fn get_ref(&self, id: &FileId) -> Result<&File, DatabaseError> {
551        let name = self.id_to_name.get(id).ok_or(DatabaseError::FileNotFound)?;
552        self.files.get(name).map(std::convert::AsRef::as_ref).ok_or(DatabaseError::FileNotFound)
553    }
554
555    #[inline]
556    fn get_by_name(&self, name: &[u8]) -> Result<Arc<File>, DatabaseError> {
557        self.files.get(name).cloned().ok_or(DatabaseError::FileNotFound)
558    }
559
560    #[inline]
561    fn get_by_path(&self, path: &Path) -> Result<Arc<File>, DatabaseError> {
562        self.files.values().find(|file| file.path.as_deref() == Some(path)).cloned().ok_or(DatabaseError::FileNotFound)
563    }
564
565    #[inline]
566    fn files(&self) -> impl Iterator<Item = Arc<File>> {
567        self.files.values().cloned()
568    }
569
570    #[inline]
571    fn len(&self) -> usize {
572        self.files.len()
573    }
574}
575
576impl DatabaseReader for ReadDatabase {
577    #[inline]
578    fn get_id(&self, name: &[u8]) -> Option<FileId> {
579        self.name_to_index.get(name).and_then(|&i| self.files.get(i)).map(|f| f.id)
580    }
581
582    #[inline]
583    fn get(&self, id: &FileId) -> Result<Arc<File>, DatabaseError> {
584        let index = self.id_to_index.get(id).ok_or(DatabaseError::FileNotFound)?;
585
586        self.files.get(*index).cloned().ok_or(DatabaseError::FileNotFound)
587    }
588
589    #[inline]
590    fn get_ref(&self, id: &FileId) -> Result<&File, DatabaseError> {
591        let index = self.id_to_index.get(id).ok_or(DatabaseError::FileNotFound)?;
592
593        self.files.get(*index).map(std::convert::AsRef::as_ref).ok_or(DatabaseError::FileNotFound)
594    }
595
596    #[inline]
597    fn get_by_name(&self, name: &[u8]) -> Result<Arc<File>, DatabaseError> {
598        self.name_to_index.get(name).and_then(|&i| self.files.get(i)).cloned().ok_or(DatabaseError::FileNotFound)
599    }
600
601    #[inline]
602    fn get_by_path(&self, path: &Path) -> Result<Arc<File>, DatabaseError> {
603        self.path_to_index.get(path).and_then(|&i| self.files.get(i)).cloned().ok_or(DatabaseError::FileNotFound)
604    }
605
606    #[inline]
607    fn files(&self) -> impl Iterator<Item = Arc<File>> {
608        self.files.iter().cloned()
609    }
610
611    #[inline]
612    fn len(&self) -> usize {
613        self.files.len()
614    }
615}
616
617#[cfg(test)]
618#[allow(clippy::unwrap_used)]
619mod tests {
620    use super::*;
621
622    fn configuration() -> DatabaseConfiguration<'static> {
623        DatabaseConfiguration::new(Path::new("/"), vec![], vec![], vec![], vec![]).into_static()
624    }
625
626    #[test]
627    fn merge_base_adds_missing_files_without_replacing_project_files() {
628        let mut base = Database::new(configuration());
629        base.add(File::new(Cow::Borrowed(b"shared.php"), FileType::Builtin, None, Cow::Borrowed(b"base")));
630        base.add(File::new(Cow::Borrowed(b"builtin.php"), FileType::Builtin, None, Cow::Borrowed(b"builtin")));
631
632        let mut project = Database::new(configuration());
633        project.add(File::new(Cow::Borrowed(b"shared.php"), FileType::Host, None, Cow::Borrowed(b"project")));
634        project.add(File::new(Cow::Borrowed(b"project.php"), FileType::Host, None, Cow::Borrowed(b"project-only")));
635
636        project.merge_base(base);
637
638        assert_eq!(project.len(), 3);
639        let shared = project.get_by_name(b"shared.php").unwrap();
640        assert_eq!(shared.file_type, FileType::Host);
641        assert_eq!(shared.contents.as_ref(), b"project");
642
643        let builtin = project.get_by_name(b"builtin.php").unwrap();
644        assert_eq!(builtin.file_type, FileType::Builtin);
645        assert_eq!(project.get(&builtin.id).unwrap().name.as_ref(), b"builtin.php");
646    }
647}
mago_database/lib.rs

mago_database/
lib.rs
