mago_database/
lib.rs

1//! High-performance file database for PHP projects.
2//!
3//! This crate provides an efficient in-memory database for managing collections of PHP source files.
4//! It offers two complementary database types optimized for different access patterns:
5//!
6//! - [`Database`]: Mutable builder optimized for modifications (add, update, delete)
7//! - [`ReadDatabase`]: Immutable snapshot optimized for high-performance reads
8//!
9//! # Architecture
10//!
11//! The database uses a two-phase approach:
12//!
13//! 1. **Build Phase**: Use [`Database`] to load files, make modifications, and track changes
14//! 2. **Query Phase**: Convert to [`ReadDatabase`] via [`Database::read_only`] for fast lookups
15//!
16//! # Key Features
17//!
18//! - **Fast Lookups**: O(1) average-time access by ID, name, or filesystem path
19//! - **Change Tracking**: Record and batch apply file modifications via [`ChangeLog`]
20//! - **Deterministic Iteration**: [`ReadDatabase`] guarantees consistent iteration order
21//! - **Parallel Operations**: Concurrent file I/O and processing support
22//! - **Type Safety**: Strong typing with stable [`FileId`] handles
23//!
24//! # Common Workflow
25//!
26//! ## Loading Files
27//!
28//! Use [`loader::DatabaseLoader`] to scan a project directory:
29//!
30//! The loader handles file discovery, exclusion patterns, and parallel loading.
31//!
32//! ## Querying Files
33//!
34//! Both database types implement [`DatabaseReader`] for uniform access:
35//!
36//! ## Modifying Files
37//!
38//! Use [`ChangeLog`] to batch modifications:
39//!
40//! Changes can be applied to the database and optionally written to disk in parallel.
41//!
42//! # Performance Characteristics
43//!
44//! ## Database (Mutable)
45//!
46//! - Add/Update/Delete: O(1) average
47//! - Lookup by ID/name: O(1) average
48//! - Iteration: Unordered
49//! - Memory: ~2x file count (maps for bidirectional lookup)
50//!
51//! ## ReadDatabase (Immutable)
52//!
53//! - Creation: O(n log n) for sorting
54//! - Lookup by ID/name/path: O(1) average
55//! - Iteration: Deterministic, sorted by FileId
56//! - Memory: ~3x file count (vector + 3 index maps)
57//!
58//! # Thread Safety
59//!
60//! [`Database`] is not thread-safe and should be used from a single thread during construction.
61//! [`ReadDatabase`] can be freely shared across threads for concurrent read access.
62
63use std::borrow::Cow;
64use std::path::Path;
65use std::path::PathBuf;
66use std::sync::Arc;
67
68use ahash::HashMap;
69use ahash::HashMapExt;
70use rayon::iter::IntoParallelIterator;
71use rayon::iter::ParallelIterator;
72use serde::Deserialize;
73use serde::Serialize;
74
75use crate::change::Change;
76use crate::change::ChangeLog;
77use crate::error::DatabaseError;
78use crate::exclusion::Exclusion;
79use crate::file::File;
80use crate::file::FileId;
81use crate::file::FileType;
82use crate::file::line_starts;
83use crate::operation::FilesystemOperation;
84
85mod utils;
86
87pub mod change;
88pub mod error;
89pub mod exclusion;
90pub mod file;
91pub mod loader;
92pub mod watcher;
93
94mod operation;
95
96/// Configuration for database loading and watching.
97#[derive(Debug, Clone, Serialize, Deserialize)]
98pub struct DatabaseConfiguration<'a> {
99    pub workspace: Cow<'a, Path>,
100    /// Paths or glob patterns for source files.
101    /// Can be directory paths (e.g., "src") or glob patterns (e.g., "src/**/*.php")
102    pub paths: Vec<Cow<'a, str>>,
103    /// Paths or glob patterns for included files.
104    /// Can be directory paths (e.g., "vendor") or glob patterns (e.g., "vendor/**/*.php")
105    pub includes: Vec<Cow<'a, str>>,
106    pub excludes: Vec<Exclusion<'a>>,
107    pub extensions: Vec<Cow<'a, str>>,
108}
109
110impl<'a> DatabaseConfiguration<'a> {
111    pub fn new(
112        workspace: &'a Path,
113        paths: Vec<&'a str>,
114        includes: Vec<&'a str>,
115        excludes: Vec<Exclusion<'a>>,
116        extensions: Vec<&'a str>,
117    ) -> Self {
118        let paths = paths.into_iter().map(Cow::Borrowed).collect();
119        let includes = includes.into_iter().map(Cow::Borrowed).collect();
120
121        let excludes = excludes
122            .into_iter()
123            .filter_map(|exclusion| match exclusion {
124                Exclusion::Path(p) => Some(if p.is_absolute() {
125                    Exclusion::Path(p)
126                } else {
127                    workspace.join(p).canonicalize().ok().map(Cow::Owned).map(Exclusion::Path)?
128                }),
129                Exclusion::Pattern(pat) => Some(Exclusion::Pattern(pat)),
130            })
131            .collect();
132
133        let extensions = extensions.into_iter().map(Cow::Borrowed).collect();
134
135        Self { workspace: Cow::Borrowed(workspace), paths, includes, excludes, extensions }
136    }
137
138    pub fn into_static(self) -> DatabaseConfiguration<'static> {
139        DatabaseConfiguration {
140            workspace: Cow::Owned(self.workspace.into_owned()),
141            paths: self.paths.into_iter().map(|s| Cow::Owned(s.into_owned())).collect(),
142            includes: self.includes.into_iter().map(|s| Cow::Owned(s.into_owned())).collect(),
143            excludes: self
144                .excludes
145                .into_iter()
146                .map(|e| match e {
147                    Exclusion::Path(p) => Exclusion::Path(Cow::Owned(p.into_owned())),
148                    Exclusion::Pattern(pat) => Exclusion::Pattern(Cow::Owned(pat.into_owned())),
149                })
150                .collect(),
151            extensions: self.extensions.into_iter().map(|s| Cow::Owned(s.into_owned())).collect(),
152        }
153    }
154}
155
156/// Mutable database for managing project files with add/update/delete operations.
157#[derive(Debug, Clone, Serialize, Deserialize)]
158pub struct Database<'a> {
159    files: HashMap<Cow<'static, str>, Arc<File>>,
160    id_to_name: HashMap<FileId, Cow<'static, str>>,
161    pub(crate) configuration: DatabaseConfiguration<'a>,
162}
163
164/// Immutable, read-optimized snapshot of the database.
165#[derive(Debug)]
166pub struct ReadDatabase {
167    files: Vec<Arc<File>>,
168    id_to_index: HashMap<FileId, usize>,
169    name_to_index: HashMap<Cow<'static, str>, usize>,
170    path_to_index: HashMap<PathBuf, usize>,
171}
172
173impl<'a> Database<'a> {
174    pub fn new(configuration: DatabaseConfiguration<'a>) -> Self {
175        Self { files: HashMap::default(), id_to_name: HashMap::default(), configuration }
176    }
177
178    pub fn single(file: File, configuration: DatabaseConfiguration<'a>) -> Self {
179        let mut db = Self::new(configuration);
180        db.add(file);
181        db
182    }
183
184    pub fn add(&mut self, file: File) -> FileId {
185        let name = file.name.clone();
186        let id = file.id;
187
188        if let Some(old_file) = self.files.insert(name.clone(), Arc::new(file)) {
189            self.id_to_name.remove(&old_file.id);
190        }
191
192        self.id_to_name.insert(id, name);
193
194        id
195    }
196
197    /// Updates a file's content in-place using its stable `FileId`.
198    ///
199    /// This recalculates derived data like file size, line endings, and `FileRevision`.
200    /// Returns `true` if a file with the given ID was found and updated.
201    pub fn update(&mut self, id: FileId, new_contents: Cow<'static, str>) -> bool {
202        if let Some(name) = self.id_to_name.get(&id)
203            && let Some(file) = self.files.get_mut(name)
204            && let Some(file) = Arc::get_mut(file)
205        {
206            file.contents = new_contents;
207            file.size = file.contents.len() as u32;
208            file.lines = line_starts(file.contents.as_ref()).collect();
209            return true;
210        }
211        false
212    }
213
214    /// Deletes a file from the database using its stable `FileId`.
215    ///
216    /// Returns `true` if a file with the given ID was found and removed.
217    pub fn delete(&mut self, id: FileId) -> bool {
218        if let Some(name) = self.id_to_name.remove(&id) { self.files.remove(&name).is_some() } else { false }
219    }
220
221    /// Commits a [`ChangeLog`], applying all its recorded operations to the database
222    /// and optionally writing them to the filesystem.
223    ///
224    /// # Arguments
225    ///
226    /// * `change_log`: The log of changes to apply.
227    /// * `write_to_disk`: If `true`, changes for files that have a filesystem
228    ///   path will be written to disk in parallel.
229    ///
230    /// # Errors
231    ///
232    /// Returns a [`DatabaseError`] if the log cannot be consumed or if any
233    /// filesystem operation fails.
234    pub fn commit(&mut self, change_log: ChangeLog, write_to_disk: bool) -> Result<(), DatabaseError> {
235        let changes = change_log.into_inner()?;
236        let mut fs_operations = if write_to_disk { Vec::new() } else { Vec::with_capacity(0) };
237
238        for change in changes {
239            match change {
240                Change::Add(file) => {
241                    if write_to_disk && let Some(path) = &file.path {
242                        fs_operations.push(FilesystemOperation::Write(path.clone(), file.contents.clone()));
243                    }
244
245                    self.add(file);
246                }
247                Change::Update(id, contents) => {
248                    if write_to_disk
249                        && let Ok(file) = self.get(&id)
250                        && let Some(path) = &file.path
251                    {
252                        fs_operations.push(FilesystemOperation::Write(path.clone(), contents.clone()));
253                    }
254
255                    self.update(id, contents);
256                }
257                Change::Delete(id) => {
258                    if write_to_disk
259                        && let Ok(file) = self.get(&id)
260                        && let Some(path) = &file.path
261                    {
262                        fs_operations.push(FilesystemOperation::Delete(path.clone()));
263                    }
264
265                    self.delete(id);
266                }
267            }
268        }
269
270        if write_to_disk {
271            fs_operations.into_par_iter().try_for_each(|op| -> Result<(), DatabaseError> { op.execute() })?;
272        }
273
274        Ok(())
275    }
276
277    /// Creates an independent, immutable snapshot of the database.
278    ///
279    /// This is a potentially expensive one-time operation as it **clones** all file
280    /// data. The resulting [`ReadDatabase`] is highly optimized for fast reads and
281    /// guarantees a deterministic iteration order. The original `Database` is not
282    /// consumed and can continue to be used.
283    pub fn read_only(&self) -> ReadDatabase {
284        let mut files_vec: Vec<Arc<File>> = self.files.values().cloned().collect();
285        files_vec.sort_unstable_by_key(|f| f.id);
286
287        let mut id_to_index = HashMap::with_capacity(files_vec.len());
288        let mut name_to_index = HashMap::with_capacity(files_vec.len());
289        let mut path_to_index = HashMap::with_capacity(files_vec.len());
290
291        for (index, file) in files_vec.iter().enumerate() {
292            id_to_index.insert(file.id, index);
293            name_to_index.insert(file.name.clone(), index);
294            if let Some(path) = &file.path {
295                path_to_index.insert(path.clone(), index);
296            }
297        }
298
299        ReadDatabase { files: files_vec, id_to_index, name_to_index, path_to_index }
300    }
301}
302
303impl ReadDatabase {
304    pub fn empty() -> Self {
305        Self {
306            files: Vec::with_capacity(0),
307            id_to_index: HashMap::with_capacity(0),
308            name_to_index: HashMap::with_capacity(0),
309            path_to_index: HashMap::with_capacity(0),
310        }
311    }
312
313    /// Creates a new `ReadDatabase` containing only a single file.
314    ///
315    /// This is a convenience constructor for situations, such as testing or
316    /// single-file tools, where an operation requires a [`DatabaseReader`]
317    /// implementation but only needs to be aware of one file.
318    ///
319    /// # Arguments
320    ///
321    /// * `file`: The single `File` to include in the database.
322    pub fn single(file: File) -> Self {
323        let mut id_to_index = HashMap::with_capacity(1);
324        let mut name_to_index = HashMap::with_capacity(1);
325        let mut path_to_index = HashMap::with_capacity(1);
326
327        id_to_index.insert(file.id, 0);
328        name_to_index.insert(file.name.clone(), 0);
329        if let Some(path) = &file.path {
330            path_to_index.insert(path.clone(), 0);
331        }
332
333        Self { files: vec![Arc::new(file)], id_to_index, name_to_index, path_to_index }
334    }
335}
336
337/// A universal interface for reading data from any database implementation.
338///
339/// This trait provides a common API for querying file data, abstracting over
340/// whether the underlying source is the mutable [`Database`] or the read-optimized
341/// [`ReadDatabase`]. This allows for writing generic code that can operate on either.
342pub trait DatabaseReader {
343    /// Retrieves a file's stable ID using its logical name.
344    fn get_id(&self, name: &str) -> Option<FileId>;
345
346    /// Retrieves a reference to a file using its stable `FileId`.
347    ///
348    /// # Errors
349    ///
350    /// Returns `DatabaseError::FileNotFound` if no file with the given ID exists.
351    fn get(&self, id: &FileId) -> Result<Arc<File>, DatabaseError>;
352
353    /// Retrieves a reference to a file using its stable `FileId`.
354    ///
355    /// # Errors
356    ///
357    /// Returns `DatabaseError::FileNotFound` if no file with the given ID exists.
358    fn get_ref(&self, id: &FileId) -> Result<&File, DatabaseError>;
359
360    /// Retrieves a reference to a file using its logical name.
361    ///
362    /// # Errors
363    ///
364    /// Returns `DatabaseError::FileNotFound` if no file with the given name exists.
365    fn get_by_name(&self, name: &str) -> Result<Arc<File>, DatabaseError>;
366
367    /// Retrieves a reference to a file by its absolute filesystem path.
368    ///
369    /// # Errors
370    ///
371    /// Returns `DatabaseError::FileNotFound` if no file with the given path exists.
372    fn get_by_path(&self, path: &Path) -> Result<Arc<File>, DatabaseError>;
373
374    /// Returns an iterator over all files in the database.
375    ///
376    /// The order is not guaranteed for `Database`, but is sorted by `FileId`
377    /// for `ReadDatabase`, providing deterministic iteration.
378    fn files(&self) -> impl Iterator<Item = Arc<File>>;
379
380    /// Returns an iterator over all files of a specific `FileType`.
381    fn files_with_type(&self, file_type: FileType) -> impl Iterator<Item = Arc<File>> {
382        self.files().filter(move |file| file.file_type == file_type)
383    }
384
385    /// Returns an iterator over all files that do not match a specific `FileType`.
386    fn files_without_type(&self, file_type: FileType) -> impl Iterator<Item = Arc<File>> {
387        self.files().filter(move |file| file.file_type != file_type)
388    }
389
390    /// Returns an iterator over the stable IDs of all files in the database.
391    fn file_ids(&self) -> impl Iterator<Item = FileId> {
392        self.files().map(|file| file.id)
393    }
394
395    /// Returns an iterator over the stable IDs of all files of a specific `FileType`.
396    fn file_ids_with_type(&self, file_type: FileType) -> impl Iterator<Item = FileId> {
397        self.files_with_type(file_type).map(|file| file.id)
398    }
399
400    /// Returns an iterator over the stable IDs of all files that do not match a specific `FileType`.
401    fn file_ids_without_type(&self, file_type: FileType) -> impl Iterator<Item = FileId> {
402        self.files_without_type(file_type).map(|file| file.id)
403    }
404
405    /// Returns the total number of files in the database.
406    fn len(&self) -> usize;
407
408    /// Returns `true` if the database contains no files.
409    fn is_empty(&self) -> bool {
410        self.len() == 0
411    }
412}
413
414impl<'a> DatabaseReader for Database<'a> {
415    fn get_id(&self, name: &str) -> Option<FileId> {
416        self.files.get(name).map(|f| f.id)
417    }
418
419    fn get(&self, id: &FileId) -> Result<Arc<File>, DatabaseError> {
420        let name = self.id_to_name.get(id).ok_or(DatabaseError::FileNotFound)?;
421        let file = self.files.get(name).ok_or(DatabaseError::FileNotFound)?;
422
423        Ok(file.clone())
424    }
425
426    fn get_ref(&self, id: &FileId) -> Result<&File, DatabaseError> {
427        let name = self.id_to_name.get(id).ok_or(DatabaseError::FileNotFound)?;
428        self.files.get(name).map(|file| file.as_ref()).ok_or(DatabaseError::FileNotFound)
429    }
430
431    fn get_by_name(&self, name: &str) -> Result<Arc<File>, DatabaseError> {
432        self.files.get(name).cloned().ok_or(DatabaseError::FileNotFound)
433    }
434
435    fn get_by_path(&self, path: &Path) -> Result<Arc<File>, DatabaseError> {
436        self.files.values().find(|file| file.path.as_deref() == Some(path)).cloned().ok_or(DatabaseError::FileNotFound)
437    }
438
439    fn files(&self) -> impl Iterator<Item = Arc<File>> {
440        self.files.values().cloned()
441    }
442
443    fn len(&self) -> usize {
444        self.files.len()
445    }
446}
447
448impl DatabaseReader for ReadDatabase {
449    fn get_id(&self, name: &str) -> Option<FileId> {
450        self.name_to_index.get(name).and_then(|&i| self.files.get(i)).map(|f| f.id)
451    }
452
453    fn get(&self, id: &FileId) -> Result<Arc<File>, DatabaseError> {
454        let index = self.id_to_index.get(id).ok_or(DatabaseError::FileNotFound)?;
455
456        self.files.get(*index).cloned().ok_or(DatabaseError::FileNotFound)
457    }
458
459    fn get_ref(&self, id: &FileId) -> Result<&File, DatabaseError> {
460        let index = self.id_to_index.get(id).ok_or(DatabaseError::FileNotFound)?;
461
462        self.files.get(*index).map(|file| file.as_ref()).ok_or(DatabaseError::FileNotFound)
463    }
464
465    fn get_by_name(&self, name: &str) -> Result<Arc<File>, DatabaseError> {
466        self.name_to_index.get(name).and_then(|&i| self.files.get(i)).cloned().ok_or(DatabaseError::FileNotFound)
467    }
468
469    fn get_by_path(&self, path: &Path) -> Result<Arc<File>, DatabaseError> {
470        self.path_to_index.get(path).and_then(|&i| self.files.get(i)).cloned().ok_or(DatabaseError::FileNotFound)
471    }
472
473    fn files(&self) -> impl Iterator<Item = Arc<File>> {
474        self.files.iter().cloned()
475    }
476
477    fn len(&self) -> usize {
478        self.files.len()
479    }
480}
mago_database/lib.rs

mago_database/
lib.rs
