Skip to main content

smeltery_prospect/
lib.rs

1#![doc = include_str!("../README.md")]
2#![cfg_attr(docsrs, feature(doc_cfg))]
3
4mod console;
5mod driver;
6mod error;
7mod highlight;
8mod hit;
9pub mod migration;
10mod search;
11mod settings;
12mod spec;
13pub mod testing;
14mod text;
15
16use std::any::TypeId;
17use std::collections::{HashMap, HashSet};
18use std::sync::{Arc, Mutex, OnceLock, PoisonError, RwLock, Weak};
19
20use smeltery_core::db::{Db, ModelChange, ModelEvent, ModelListener, Record};
21use smeltery_core::{App, AppBuilder, BoxFuture, Error, Result};
22
23pub use error::ProspectError;
24pub use highlight::{Highlight, Highlights, Segment};
25pub use hit::Hit;
26pub use search::{Direction, FilterValue, Search};
27pub use settings::{Driver, ProspectSettings};
28pub use spec::{IndexSpec, Language, TextColumn, Weight};
29pub use text::{MAX_TERMS, MIN_PREFIX, MYSQL_STOPWORDS, SearchText};
30
31use driver::memory::MemoryEngine;
32use spec::Resolved;
33
34/// A model that can be searched: implement it next to the model's entity (`app/models/post.rs`) and register the
35/// model with `.prospect(|p| { p.model::<Post>(); })`.
36///
37/// The model needs one integer primary key (`t.id()`), and its search index needs a migration
38/// ([`migration::SearchIndex`]).
39pub trait Searchable:
40    Record + Clone + sea_orm::ModelTrait<Entity = <Self as Record>::Entity> + sea_orm::FromQueryResult
41{
42    /// Declare the searched, filtered and sorted columns (see [`IndexSpec`]).
43    fn index(i: &mut IndexSpec);
44
45    /// A search of this model for `text` (parsed with [`SearchText::parse`]); refine it with the [`Search`] methods
46    /// and run it with `paginate`, `get`, `keys` or `count`.
47    fn search(prospect: &Prospect, text: &str) -> Search<Self> {
48        Search::new(prospect, text)
49    }
50}
51
52/// The registration of searchable models, inside [`ProspectExt::prospect`].
53#[derive(Default)]
54pub struct Models {
55    entries: Vec<std::result::Result<(TypeId, Resolved), ProspectError>>,
56}
57
58impl std::fmt::Debug for Models {
59    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
60        f.debug_struct("Models")
61            .field("count", &self.entries.len())
62            .finish()
63    }
64}
65
66impl Models {
67    /// Make `M` searchable. Its [`IndexSpec`] is checked against its entity; a mistake stops the app's build with
68    /// the table and the column.
69    pub fn model<M: Searchable>(&mut self) -> &mut Self {
70        self.entries
71            .push(spec::resolve::<M>().map(|r| (TypeId::of::<M>(), r)));
72        self
73    }
74}
75
76/// Installs Prospect on an [`AppBuilder`] (`bootstrap/app.rs`).
77pub trait ProspectExt: Sized {
78    /// Install Prospect with the settings from `.env` ([`ProspectSettings::from_env`]) and the models `register`
79    /// names: the [`Prospect`] service (a handler argument), and the console commands `prospect:status`,
80    /// `prospect:import` and `prospect:flush`. The build fails on an invalid spec, a model registered twice, two
81    /// models with one index name, or an unknown `PROSPECT_DRIVER`.
82    fn prospect(self, register: impl FnOnce(&mut Models)) -> Self;
83
84    /// Like [`prospect`](Self::prospect), with these settings.
85    fn prospect_with(self, settings: ProspectSettings, register: impl FnOnce(&mut Models)) -> Self;
86}
87
88impl ProspectExt for AppBuilder {
89    fn prospect(self, register: impl FnOnce(&mut Models)) -> Self {
90        self.prospect_with(ProspectSettings::from_env(), register)
91    }
92
93    fn prospect_with(self, settings: ProspectSettings, register: impl FnOnce(&mut Models)) -> Self {
94        let mut models = Models::default();
95        register(&mut models);
96        let mut problem: Option<ProspectError> = None;
97        let driver = match settings.checked_driver() {
98            Ok(driver) => driver,
99            Err(e) => {
100                problem = Some(e);
101                Driver::Database
102            }
103        };
104        let mut specs: HashMap<TypeId, Arc<Resolved>> = HashMap::new();
105        let mut names: HashSet<String> = HashSet::new();
106        for entry in models.entries {
107            match entry {
108                Err(e) => {
109                    problem.get_or_insert(e);
110                }
111                Ok((type_id, resolved)) => {
112                    if specs.contains_key(&type_id) || !names.insert(resolved.index.clone()) {
113                        problem.get_or_insert(ProspectError::Spec {
114                            table: resolved.table,
115                            reason: format!(
116                                "the index `{}` is registered twice (a model registered twice, or two models with one \
117                                 index name)",
118                                resolved.index
119                            ),
120                        });
121                        continue;
122                    }
123                    specs.insert(type_id, Arc::new(resolved));
124                }
125            }
126        }
127        // The memory engine is written by model events. They are listened to only when that engine can be used: with
128        // `PROSPECT_DRIVER=memory`, or under `APP_ENV=testing` (where `testing::fake` switches to it). The database
129        // driver needs no events, so a production app with it registers no listener at all.
130        let listens = driver == Driver::Memory || self.settings().env == "testing";
131        let inner = Arc::new(Inner {
132            settings,
133            driver: RwLock::new(driver),
134            memory: RwLock::new(Arc::new(MemoryEngine::default())),
135            models: specs,
136            db: OnceLock::new(),
137            checked: Mutex::new(HashSet::new()),
138            reported: Mutex::new(HashSet::new()),
139            mysql_min_token: tokio::sync::OnceCell::new(),
140            listens,
141        });
142        let prospect = Prospect {
143            inner: Arc::clone(&inner),
144        };
145        let mut builder = self.service(prospect.clone());
146        if listens {
147            builder = builder.model_listener(SyncListener(Arc::downgrade(&inner)));
148        }
149        builder
150            .commands(|c| {
151                c.add(console::Status);
152                c.add(console::Import);
153                c.add(console::Flush);
154            })
155            .on_boot(move |app| async move {
156                if let Some(e) = problem {
157                    return Err(e.into());
158                }
159                if let Ok(db) = app.db() {
160                    let _ = prospect.inner.db.set(db);
161                }
162                Ok(())
163            })
164            .on_serve(|app| async move {
165                if let Ok(prospect) = Prospect::of(&app) {
166                    prospect.report_missing_indexes().await;
167                }
168                Ok(())
169            })
170    }
171}
172
173/// The search service: a handler argument (`prospect: Prospect`), or [`Prospect::of`] elsewhere. Cheap to clone.
174#[derive(Clone)]
175pub struct Prospect {
176    inner: Arc<Inner>,
177}
178
179impl std::fmt::Debug for Prospect {
180    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
181        f.debug_struct("Prospect")
182            .field("driver", &self.driver())
183            .field("models", &self.inner.models.len())
184            .finish_non_exhaustive()
185    }
186}
187
188pub(crate) struct Inner {
189    pub(crate) settings: ProspectSettings,
190    driver: RwLock<Driver>,
191    memory: RwLock<Arc<MemoryEngine>>,
192    pub(crate) models: HashMap<TypeId, Arc<Resolved>>,
193    db: OnceLock<Db>,
194    /// Models whose database index was found as their spec wants it.
195    checked: Mutex<HashSet<TypeId>>,
196    /// Models whose missing index was logged already.
197    reported: Mutex<HashSet<TypeId>>,
198    /// `innodb_ft_min_token_size`, read once.
199    pub(crate) mysql_min_token: tokio::sync::OnceCell<usize>,
200    listens: bool,
201}
202
203impl Prospect {
204    /// The app's Prospect.
205    ///
206    /// # Errors
207    /// Prospect is not installed (`.prospect(…)` in `bootstrap/app.rs`).
208    pub fn of(app: &App) -> Result<Self> {
209        app.service::<Prospect>()
210            .map(|p| (*p).clone())
211            .ok_or_else(|| {
212                Error::internal(
213                    "Prospect is not installed: add `.prospect(|p| { … })` in bootstrap/app.rs",
214                )
215            })
216    }
217
218    /// The driver that answers searches.
219    pub fn driver(&self) -> Driver {
220        *self
221            .inner
222            .driver
223            .read()
224            .unwrap_or_else(PoisonError::into_inner)
225    }
226
227    /// The settings.
228    pub fn settings(&self) -> &ProspectSettings {
229        &self.inner.settings
230    }
231
232    /// Run `fut` with Prospect's model listener (and every other model listener) switched off for the writes it
233    /// makes in this task; with an engine, call [`import`](Self::import) afterwards. The database driver keeps its
234    /// index current inside the database, so it is not affected.
235    pub async fn paused<F: std::future::Future>(&self, fut: F) -> F::Output {
236        smeltery_core::db::without_listeners(fut).await
237    }
238
239    /// Fill the index of `M` from its table, returning the rows indexed. On SQLite the database driver rebuilds the
240    /// FTS5 index from the table (`'rebuild'`); PostgreSQL and MySQL keep their index current themselves (0). The
241    /// memory engine reads every row in chunks of `PROSPECT_BATCH`.
242    ///
243    /// # Errors
244    /// `M` is not registered, there is no database, or a statement fails.
245    pub async fn import<M: Searchable>(&self) -> Result<u64> {
246        let spec = self.spec_of::<M>()?;
247        let db = self.db()?;
248        match self.driver() {
249            Driver::Database => driver::database::rebuild(&db, &spec).await,
250            Driver::Memory => driver::memory::import::<M>(self, &db, &spec).await,
251        }
252    }
253
254    /// Bring the documents of these keys of `M` up to date: for writes that do not go through `Record` (raw SQL,
255    /// SeaORM's `update_many`). The database driver needs nothing (its index follows the table); the memory engine
256    /// re-reads each row (indexed when it exists and its `only_when` column is true, removed otherwise).
257    ///
258    /// # Errors
259    /// `M` is not registered, there is no database, or a query fails.
260    pub async fn sync<M: Searchable>(&self, keys: impl IntoIterator<Item = i64>) -> Result<()> {
261        let spec = self.spec_of::<M>()?;
262        if self.driver() == Driver::Database {
263            return Ok(());
264        }
265        let db = self.db()?;
266        let keys: Vec<i64> = keys.into_iter().collect();
267        driver::memory::sync::<M>(self, &db, &spec, &keys).await
268    }
269
270    /// Remove every document of `M` from the engine. Refused by the database driver: its index follows the table.
271    ///
272    /// # Errors
273    /// `M` is not registered, or the driver is `database`.
274    pub async fn flush<M: Searchable>(&self) -> Result<()> {
275        let spec = self.spec_of::<M>()?;
276        self.flush_spec(&spec)
277    }
278
279    pub(crate) fn flush_spec(&self, spec: &Resolved) -> Result<()> {
280        match self.driver() {
281            Driver::Database => Err(ProspectError::Unsupported(
282                "the database index follows the table: there is nothing to flush".to_owned(),
283            )
284            .into()),
285            Driver::Memory => {
286                self.memory().flush(&spec.index);
287                Ok(())
288            }
289        }
290    }
291
292    pub(crate) fn spec_of<M: Searchable>(
293        &self,
294    ) -> std::result::Result<Arc<Resolved>, ProspectError> {
295        self.inner
296            .models
297            .get(&TypeId::of::<M>())
298            .cloned()
299            .ok_or(ProspectError::NotRegistered)
300    }
301
302    pub(crate) fn db(&self) -> Result<Db> {
303        self.inner
304            .db
305            .get()
306            .cloned()
307            .ok_or_else(|| Error::internal("Prospect needs a database: set DATABASE_URL in .env"))
308    }
309
310    pub(crate) fn memory(&self) -> Arc<MemoryEngine> {
311        Arc::clone(
312            &self
313                .inner
314                .memory
315                .read()
316                .unwrap_or_else(PoisonError::into_inner),
317        )
318    }
319
320    /// Switch to a fresh memory engine (the testing fake).
321    pub(crate) fn use_fresh_memory(&self) -> std::result::Result<Arc<MemoryEngine>, ProspectError> {
322        if !self.inner.listens {
323            return Err(ProspectError::Unsupported(
324                "the memory engine needs APP_ENV=testing or PROSPECT_DRIVER=memory when the app builds".to_owned(),
325            ));
326        }
327        let engine = Arc::new(MemoryEngine::default());
328        *self
329            .inner
330            .memory
331            .write()
332            .unwrap_or_else(PoisonError::into_inner) = Arc::clone(&engine);
333        *self
334            .inner
335            .driver
336            .write()
337            .unwrap_or_else(PoisonError::into_inner) = Driver::Memory;
338        Ok(engine)
339    }
340
341    /// Check that the database has `spec`'s index as the spec wants it; a passed check is remembered, a failure is
342    /// logged once per model (ERROR) and answered as [`ProspectError::IndexMissing`].
343    pub(crate) async fn ensure_index(
344        &self,
345        type_id: TypeId,
346        spec: &Resolved,
347        db: &Db,
348    ) -> Result<()> {
349        if lock(&self.inner.checked).contains(&type_id) {
350            return Ok(());
351        }
352        match driver::database::check_index(self, db, spec).await? {
353            None => {
354                lock(&self.inner.checked).insert(type_id);
355                Ok(())
356            }
357            Some(reason) => {
358                if lock(&self.inner.reported).insert(type_id) {
359                    tracing::error!(
360                        table = spec.table,
361                        reason = %reason,
362                        "prospect: the search index is missing or does not match the spec; write a migration with \
363                         SearchIndex::on(..) and run `migrate`"
364                    );
365                }
366                Err(ProspectError::IndexMissing {
367                    table: spec.table,
368                    reason,
369                }
370                .into())
371            }
372        }
373    }
374
375    /// Every registered model's index state, by table name.
376    pub(crate) async fn statuses(
377        &self,
378    ) -> Vec<(Arc<Resolved>, Result<Option<String>>, Option<u64>)> {
379        let mut specs: Vec<Arc<Resolved>> = self.inner.models.values().cloned().collect();
380        specs.sort_by(|a, b| a.table.cmp(b.table));
381        let mut out = Vec::new();
382        for spec in specs {
383            let db = self.db();
384            let (state, rows) = match &db {
385                Ok(db) => (
386                    driver::database::check_index(self, db, &spec).await,
387                    driver::database::row_count(db, &spec).await.ok(),
388                ),
389                Err(_) => (Err(Error::internal("no database")), None),
390            };
391            out.push((spec, state, rows));
392        }
393        out
394    }
395
396    async fn report_missing_indexes(&self) {
397        if self.driver() != Driver::Database {
398            return;
399        }
400        for (spec, state, _) in self.statuses().await {
401            match state {
402                Ok(None) => {}
403                Ok(Some(reason)) => tracing::error!(
404                    table = spec.table,
405                    reason = %reason,
406                    "prospect: the search index is missing or does not match the spec; write a migration with \
407                     SearchIndex::on(..) and run `migrate`"
408                ),
409                Err(e) => {
410                    tracing::error!(table = spec.table, error = %e, "prospect: the search index could not be checked")
411                }
412            }
413        }
414    }
415}
416
417pub(crate) fn lock<T>(m: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
418    m.lock().unwrap_or_else(PoisonError::into_inner)
419}
420
421impl axum::extract::FromRequestParts<App> for Prospect {
422    type Rejection = Error;
423
424    async fn from_request_parts(
425        _parts: &mut http::request::Parts,
426        app: &App,
427    ) -> std::result::Result<Self, Self::Rejection> {
428        Prospect::of(app)
429    }
430}
431
432/// Writes the memory engine from model events (registered only when that engine can be used).
433struct SyncListener(Weak<Inner>);
434
435impl ModelListener for SyncListener {
436    fn changed<'a>(&'a self, _db: &'a Db, event: ModelEvent<'a>) -> BoxFuture<'a, ()> {
437        Box::pin(async move {
438            let Some(inner) = self.0.upgrade() else {
439                return;
440            };
441            let Some(spec) = inner.models.get(&event.model_type) else {
442                return;
443            };
444            let prospect = Prospect {
445                inner: Arc::clone(&inner),
446            };
447            if prospect.driver() != Driver::Memory {
448                return;
449            }
450            let Some((key, document)) = (spec.document_of)(event.row, spec) else {
451                return;
452            };
453            let engine = prospect.memory();
454            match (event.change, document) {
455                (ModelChange::Deleted, _) | (_, None) => engine.delete(&spec.index, key),
456                (_, Some(document)) => engine.upsert(&spec.index, key, document),
457            }
458        })
459    }
460}