Skip to main content

meerkat_sqlite/
watch.rs

1//! Cross-process change watch for one SQLite database file.
2//!
3//! Another process's commit to a database is invisible to this process's
4//! in-memory signals. This watch turns the file-system notifications that can
5//! announce a commit (on the database, its `-wal` sidecar and their
6//! directory) into coalesced ticks for a consumer that then reads durable
7//! state. A connection's own open and close are no change and tick nothing
8//! ([`sqlite_watch_event_relevant`]): otherwise a consumer's own read would
9//! wake it again.
10//!
11//! A notification is a HINT, never a guarantee: SQLite writes WAL frames
12//! before the WAL index makes the commit visible, so a reader woken by an
13//! event can still read the old state, and no later event is promised. The
14//! watch therefore also ticks on a bounded sweep interval; the consumer's
15//! sweep read is the eventual-wake guarantee. Neither the watch nor the sweep
16//! ever defines permission, custody or expiry: correctness rests with the
17//! consumer's durable reads and its own fences.
18//!
19//! Registering for notifications is a hint too, and nothing waits for it. The
20//! operating system can hold a registration without bound (macOS FSEvents
21//! registration has taken over 30 seconds) and nothing can interrupt it, so
22//! the sweep worker starts first and the registration runs on a thread of its
23//! own: starting a watch returns at once, and a registration that fails leaves
24//! the watch on its sweep alone. [`SqliteChangeWatch::registration`] reports
25//! where it stands.
26//!
27//! This is the contract the mob event bus has used since its external watch
28//! landed, extracted so every store shares one implementation.
29//!
30//! Teardown is explicit. Dropping a watch, or [`SqliteWatchStopHandle::request_stop`],
31//! only asks the worker to return and never blocks: a tick already running
32//! finishes on its own thread, unjoined. [`SqliteChangeWatch::stop_and_join`]
33//! is the one operation that waits for the worker, and it cannot do so from
34//! the worker's own thread. Neither waits for a registration that is still
35//! running: a stop cancels it, and its thread discards its result when the
36//! platform call returns. That thread is never detached: it stays with its
37//! watch and, once the watch is dropped, with the process
38//! ([`retained_registrations`]) until it returns.
39
40use std::panic::AssertUnwindSafe;
41use std::path::{Path, PathBuf};
42use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
43use std::sync::{Arc, Mutex, MutexGuard, OnceLock, PoisonError, mpsc};
44use std::thread;
45use std::time::{Duration, Instant};
46
47use notify::{RecursiveMode, Watcher as _};
48
49/// Default sweep interval: the bounded eventual-wake guarantee.
50pub const SQLITE_WATCH_SWEEP: Duration = Duration::from_millis(5_000);
51/// Settle delay after a notification, so a burst of events (a commit touches
52/// the WAL and its index) coalesces into one tick.
53const SQLITE_WATCH_COALESCE: Duration = Duration::from_millis(10);
54const SQLITE_WATCH_RECOVERY_BACKOFF_MIN: Duration = Duration::from_millis(100);
55const SQLITE_WATCH_RECOVERY_BACKOFF_MAX: Duration = Duration::from_millis(30_000);
56/// Notification registrations a process runs at once. A stalled platform
57/// (macOS FSEvents) holds each one for as long as it stalls, with two threads
58/// apiece, so a stall holds at most 128 threads; past this many, a new watch
59/// runs on its sweep alone instead of adding two more. A watch refused at the
60/// cap never registers later: it stays on its sweep for the rest of its life,
61/// which for a store held open is the life of the process. That is still
62/// correct, since correctness rests on the sweep and a notification only
63/// wakes the consumer sooner. The cap fits the bursts of a normal startup,
64/// such as parallel store opens. A queue that registers a refused watch once
65/// a place frees was considered and closed: the cliff needs more than 64
66/// registrations in flight at once.
67const MAX_REGISTRATIONS_IN_FLIGHT: usize = 64;
68/// Registration threads running now in this process, for watches alive or
69/// dropped.
70static REGISTRATIONS_IN_FLIGHT: AtomicUsize = AtomicUsize::new(0);
71
72/// Why the consumer is being asked to look.
73#[derive(Debug, Clone, Copy, PartialEq, Eq)]
74pub enum SqliteWatchTick {
75    /// A relevant file-system notification arrived (coalesced).
76    Changed,
77    /// The sweep interval elapsed without a notification.
78    Sweep,
79}
80
81/// What the consumer's tick achieved.
82#[derive(Debug, Clone, Copy, PartialEq, Eq)]
83pub enum SqliteWatchControl {
84    /// The tick was handled.
85    Handled,
86    /// The consumer's durable read failed: ticks are suppressed until a
87    /// bounded exponential backoff elapses, so a notification storm cannot
88    /// hammer a failing store.
89    Failed,
90    /// The consumer is gone: stop the watch.
91    Stop,
92}
93
94/// How [`SqliteChangeWatch::stop_and_join`] ended.
95#[derive(Debug, Clone, Copy, PartialEq, Eq)]
96#[must_use]
97pub enum SqliteWatchJoin {
98    /// The worker returned and was joined: no tick is running and none will
99    /// run.
100    Joined,
101    /// The worker was joined after a tick panicked: no tick is running and
102    /// none will run.
103    JoinedAfterPanic,
104    /// Called on the worker's own thread, from inside a tick. The stop is
105    /// requested, but a thread cannot join itself: the worker returns after
106    /// the current tick, unjoined. The join handle stays in this watch, so
107    /// whoever holds the watch must keep it and join it from another thread.
108    /// Never a settled outcome.
109    UnjoinedOnWorkerThread,
110}
111
112/// Where a watch's notification registration stands
113/// ([`SqliteChangeWatch::registration`]). Notifications only shorten a wake:
114/// the sweep runs whatever this says.
115#[derive(Debug, Clone, PartialEq, Eq)]
116#[non_exhaustive]
117pub enum SqliteWatchRegistration {
118    /// Running on its own thread, which the operating system can hold without
119    /// bound. Until it ends, only the sweep wakes the consumer.
120    Pending,
121    /// Registered. Its notifications stop when the watch does.
122    Registered,
123    /// Failed, or its thread could not start: only the sweep wakes the
124    /// consumer.
125    Failed(String),
126    /// The watch stopped before the registration ended; its thread discarded
127    /// the result.
128    Cancelled,
129}
130
131/// Asks a watch's worker to return, without owning the worker. Cheap to clone;
132/// never blocks and never joins.
133#[derive(Debug, Clone)]
134pub struct SqliteWatchStopHandle {
135    control: Arc<WatchControl>,
136}
137
138impl SqliteWatchStopHandle {
139    /// Ask the worker to return. Once it observes the request it starts no
140    /// further tick; a tick already running finishes first. Never blocks.
141    pub fn request_stop(&self) {
142        self.control.request_stop();
143    }
144}
145
146#[derive(Debug)]
147struct WatchControl {
148    stop_requested: Arc<AtomicBool>,
149    wake: mpsc::Sender<()>,
150}
151
152impl WatchControl {
153    fn request_stop(&self) {
154        self.stop_requested.store(true, Ordering::Release);
155        // Wake a worker waiting for its next tick. A worker that already
156        // returned has dropped the receiver; nothing is left to wake.
157        let _ = self.wake.send(());
158    }
159}
160
161/// Join custody of a watch's worker thread.
162enum WorkerCustody {
163    Running(thread::JoinHandle<()>),
164    Joined(SqliteWatchJoin),
165}
166
167/// A registration's result, shared by its thread and its watch.
168enum RegistrationSlot {
169    Pending,
170    /// Registered; the source until the watch stops.
171    Registered(Option<Box<dyn WatchSource>>),
172    Failed(String),
173    Cancelled,
174}
175
176/// One running registration's place among [`MAX_REGISTRATIONS_IN_FLIGHT`],
177/// given back when its thread ends, or when the thread never starts.
178struct InFlightPlace;
179
180impl InFlightPlace {
181    /// Take a place, or say how many registrations hold them all.
182    fn take() -> Result<Self, usize> {
183        REGISTRATIONS_IN_FLIGHT
184            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |running| {
185                (running < MAX_REGISTRATIONS_IN_FLIGHT).then_some(running + 1)
186            })
187            .map(|_| Self)
188    }
189}
190
191impl Drop for InFlightPlace {
192    fn drop(&mut self) {
193        REGISTRATIONS_IN_FLIGHT.fetch_sub(1, Ordering::AcqRel);
194    }
195}
196
197/// A registration thread and what it registers.
198struct RegistrationThread {
199    handle: thread::JoinHandle<()>,
200    directory: PathBuf,
201    name: String,
202    started: Instant,
203}
204
205impl RegistrationThread {
206    fn describe(&self) -> SqliteWatchRetainedRegistration {
207        SqliteWatchRetainedRegistration {
208            directory: self.directory.clone(),
209            thread_name: self.name.clone(),
210            running_for: self.started.elapsed(),
211        }
212    }
213}
214
215/// Custody of a watch's notification registration: its result, and its
216/// thread until that is joined.
217struct RegistrationCustody {
218    slot: Arc<Mutex<RegistrationSlot>>,
219    /// `None` once joined, or when the thread never started.
220    thread: Option<RegistrationThread>,
221}
222
223impl RegistrationCustody {
224    /// Register `targets` on a thread of its own, named after the watch. A
225    /// thread that cannot start, or a process already running
226    /// [`MAX_REGISTRATIONS_IN_FLIGHT`] registrations, leaves the registration
227    /// failed: the sweep alone wakes the consumer, for the rest of the
228    /// watch's life.
229    fn spawn(
230        watch_name: &str,
231        backend: Box<dyn WatchBackend>,
232        targets: WatchTargets,
233        wake: mpsc::Sender<()>,
234        stop_requested: Arc<AtomicBool>,
235    ) -> Self {
236        let slot = Arc::new(Mutex::new(RegistrationSlot::Pending));
237        let place = match InFlightPlace::take() {
238            Ok(place) => place,
239            Err(running) => {
240                settle_failed(
241                    &slot,
242                    format!(
243                        "{running} sqlite change watch notification registrations are already \
244                         pending (at most {MAX_REGISTRATIONS_IN_FLIGHT})"
245                    ),
246                );
247                return Self { slot, thread: None };
248            }
249        };
250        let thread_slot = Arc::clone(&slot);
251        let directory = targets.parent.clone();
252        let name = format!("{watch_name}-register");
253        let started = Instant::now();
254        let spawned = thread::Builder::new().name(name.clone()).spawn(move || {
255            // Given back as the thread ends, however it ends.
256            let _place = place;
257            register(
258                backend.as_ref(),
259                &targets,
260                wake,
261                &stop_requested,
262                &thread_slot,
263            );
264        });
265        match spawned {
266            Ok(handle) => Self {
267                slot,
268                thread: Some(RegistrationThread {
269                    handle,
270                    directory,
271                    name,
272                    started,
273                }),
274            },
275            Err(error) => {
276                settle_failed(
277                    &slot,
278                    format!("cannot start the sqlite change watch registration thread: {error}"),
279                );
280                Self { slot, thread: None }
281            }
282        }
283    }
284
285    fn state(&self) -> SqliteWatchRegistration {
286        match &*lock(&self.slot) {
287            RegistrationSlot::Pending => SqliteWatchRegistration::Pending,
288            RegistrationSlot::Registered(_) => SqliteWatchRegistration::Registered,
289            RegistrationSlot::Failed(reason) => SqliteWatchRegistration::Failed(reason.clone()),
290            RegistrationSlot::Cancelled => SqliteWatchRegistration::Cancelled,
291        }
292    }
293
294    /// The registration's thread while the registration is pending. Its
295    /// thread settles the slot before it ends, so a pending registration's
296    /// thread is running.
297    fn pending(&self) -> Option<SqliteWatchRetainedRegistration> {
298        if !matches!(*lock(&self.slot), RegistrationSlot::Pending) {
299            return None;
300        }
301        self.thread.as_ref().map(RegistrationThread::describe)
302    }
303
304    /// Stop the notifications, once the stop flag is set: unregister a
305    /// registered source ([`unregister`]), and join the thread if it has
306    /// returned. A registration still running discards its result when it
307    /// ends.
308    fn stop(&mut self) {
309        let source = match &mut *lock(&self.slot) {
310            RegistrationSlot::Registered(source) => source.take(),
311            _ => None,
312        };
313        if let Some(source) = source {
314            unregister(source);
315        }
316        if self
317            .thread
318            .as_ref()
319            .is_some_and(|thread| thread.handle.is_finished())
320            && let Some(thread) = self.thread.take()
321        {
322            let _ = thread.handle.join();
323        }
324    }
325
326    /// The watch is going: stop, and hand a thread still running to the
327    /// process's custody.
328    fn release(&mut self) {
329        self.stop();
330        if let Some(thread) = self.thread.take() {
331            retain_registration(thread);
332        }
333    }
334}
335
336/// A running watch: custody of its worker thread and of its notification
337/// registration.
338///
339/// Dropping it requests the stop and never joins its worker: the worker
340/// returns at its next wake, after any tick that is running, unjoined. Use
341/// [`Self::stop_and_join`] to wait for the worker.
342pub struct SqliteChangeWatch {
343    control: Arc<WatchControl>,
344    registration: RegistrationCustody,
345    worker: WorkerCustody,
346}
347
348impl std::fmt::Debug for SqliteChangeWatch {
349    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
350        f.debug_struct("SqliteChangeWatch").finish_non_exhaustive()
351    }
352}
353
354impl SqliteChangeWatch {
355    /// A handle that can request this watch's stop after the watch itself has
356    /// moved elsewhere.
357    pub fn stop_handle(&self) -> SqliteWatchStopHandle {
358        SqliteWatchStopHandle {
359            control: Arc::clone(&self.control),
360        }
361    }
362
363    /// Ask the worker to return; see [`SqliteWatchStopHandle::request_stop`].
364    pub fn request_stop(&self) {
365        self.control.request_stop();
366    }
367
368    /// Whether the calling thread is this watch's worker, i.e. the caller is
369    /// running inside one of its ticks.
370    pub fn on_worker_thread(&self) -> bool {
371        self.worker_thread_id() == Some(thread::current().id())
372    }
373
374    /// The worker thread's identity until the worker is joined.
375    pub fn worker_thread_id(&self) -> Option<thread::ThreadId> {
376        match &self.worker {
377            WorkerCustody::Running(worker) => Some(worker.thread().id()),
378            WorkerCustody::Joined(_) => None,
379        }
380    }
381
382    /// Whether the worker has returned (joined or not). A returned worker
383    /// runs no tick again.
384    pub fn is_finished(&self) -> bool {
385        match &self.worker {
386            WorkerCustody::Running(worker) => worker.is_finished(),
387            WorkerCustody::Joined(_) => true,
388        }
389    }
390
391    /// Where the notification registration stands. The sweep runs whatever
392    /// it says.
393    pub fn registration(&self) -> SqliteWatchRegistration {
394        self.registration.state()
395    }
396
397    /// The notification registration this watch would leave with the
398    /// process if it were dropped now, described as
399    /// [`retained_registrations`] would list it: `Some` exactly while
400    /// [`Self::registration`] is [`SqliteWatchRegistration::Pending`].
401    ///
402    /// Read after a stop request, it names the registration the stop
403    /// cancelled while the operating system still held it: that thread
404    /// discards the result when the platform call returns, and until then
405    /// stays with this watch or, once the watch is dropped, with the
406    /// process.
407    pub fn pending_registration(&self) -> Option<SqliteWatchRetainedRegistration> {
408        self.registration.pending()
409    }
410
411    /// Request the stop, stop the notifications, and wait for the worker to
412    /// return: a tick that is running finishes first. Unbounded by design: it
413    /// waits exactly as long as that tick runs.
414    ///
415    /// It never waits for a notification registration that is still running:
416    /// the stop cancels it, and its thread stays in this watch until it
417    /// returns ([`Self::registration`] says
418    /// [`SqliteWatchRegistration::Pending`] until then). A later call joins
419    /// that thread once it has.
420    ///
421    /// On the worker's own thread it cannot wait; it returns
422    /// [`SqliteWatchJoin::UnjoinedOnWorkerThread`] and keeps the join handle
423    /// in this watch, so a later call from another thread still joins.
424    /// Calling it again after a join returns the recorded outcome.
425    pub fn stop_and_join(&mut self) -> SqliteWatchJoin {
426        self.control.request_stop();
427        self.registration.stop();
428        if self.on_worker_thread() {
429            return SqliteWatchJoin::UnjoinedOnWorkerThread;
430        }
431        let outcome = match std::mem::replace(
432            &mut self.worker,
433            WorkerCustody::Joined(SqliteWatchJoin::Joined),
434        ) {
435            WorkerCustody::Running(worker) => match worker.join() {
436                Ok(()) => SqliteWatchJoin::Joined,
437                Err(_) => SqliteWatchJoin::JoinedAfterPanic,
438            },
439            WorkerCustody::Joined(outcome) => outcome,
440        };
441        self.worker = WorkerCustody::Joined(outcome);
442        outcome
443    }
444}
445
446impl Drop for SqliteChangeWatch {
447    fn drop(&mut self) {
448        // Never wait here: the last owner can drop a watch on any thread,
449        // including its own worker's. A registration still running goes to
450        // the process's custody.
451        self.control.request_stop();
452        self.registration.release();
453    }
454}
455
456/// Registration threads still running after their watch was dropped.
457static RETAINED_REGISTRATIONS: OnceLock<Mutex<Vec<RegistrationThread>>> = OnceLock::new();
458
459/// The process's unregister thread, `sqlite-watch-unregister`: the queue it
460/// drops sources from, in turn, and the thread itself, which lives as long as
461/// the process. `None` when it could not start.
462static UNREGISTER: OnceLock<Option<Unregister>> = OnceLock::new();
463
464struct Unregister {
465    queue: mpsc::Sender<Box<dyn WatchSource>>,
466    _thread: thread::JoinHandle<()>,
467}
468
469/// Drop a stopped watch's notification source. A source whose drop waits on
470/// the platform ([`WatchSource::teardown_waits`]) goes to the process's one
471/// unregister thread, which drops such sources in turn, so no thread that
472/// stops a watch (a runtime worker shutting down, a watch's own worker) waits
473/// on the platform. However many watches stop at once, that is one thread;
474/// their sources wait in its queue. Until the thread drops it, a source can
475/// still notify, which only wakes its own stopped watch: its worker returns
476/// at that wake without a tick, and a watch started meanwhile on the same
477/// database has its own source and wake. Any other source, or every source
478/// when the thread cannot start, is dropped here.
479fn unregister(source: Box<dyn WatchSource>) {
480    if !source.teardown_waits() {
481        drop(source);
482        return;
483    }
484    let unregister = UNREGISTER.get_or_init(|| {
485        let (queue, sources) = mpsc::channel::<Box<dyn WatchSource>>();
486        thread::Builder::new()
487            .name("sqlite-watch-unregister".to_string())
488            .spawn(move || {
489                for source in sources {
490                    // A drop can panic (notify's macOS watcher expects its
491                    // run-loop thread to join). Contain it, as a registration's
492                    // panic is, so this thread keeps dropping its queue.
493                    if std::panic::catch_unwind(AssertUnwindSafe(move || drop(source))).is_err() {
494                        tracing::warn!(
495                            "dropping a stopped sqlite change watch's notification source panicked"
496                        );
497                    }
498                }
499            })
500            .map(|thread| Unregister {
501                queue,
502                _thread: thread,
503            })
504            .ok()
505    });
506    let source = match unregister {
507        Some(unregister) => match unregister.queue.send(source) {
508            Ok(()) => return,
509            Err(mpsc::SendError(source)) => source,
510        },
511        None => source,
512    };
513    drop(source);
514}
515
516fn retained() -> &'static Mutex<Vec<RegistrationThread>> {
517    RETAINED_REGISTRATIONS.get_or_init(Mutex::default)
518}
519
520/// A notification registration still running after its watch was dropped
521/// ([`retained_registrations`]), or one a live watch would leave with the
522/// process ([`SqliteChangeWatch::pending_registration`]).
523#[derive(Debug, Clone, PartialEq, Eq)]
524#[non_exhaustive]
525pub struct SqliteWatchRetainedRegistration {
526    /// The directory it registers.
527    pub directory: PathBuf,
528    /// The name of its thread.
529    pub thread_name: String,
530    /// How long it has been running.
531    pub running_for: Duration,
532}
533
534/// The notification registrations this process still holds for watches that
535/// are gone. Each one is blocked in the operating system, which nothing can
536/// interrupt, and discards its result when that call returns. Joins the ones
537/// that have returned; never waits.
538pub fn retained_registrations() -> Vec<SqliteWatchRetainedRegistration> {
539    let mut retained = lock(retained());
540    reap(&mut retained);
541    retained.iter().map(RegistrationThread::describe).collect()
542}
543
544fn retain_registration(thread: RegistrationThread) {
545    let mut retained = lock(retained());
546    reap(&mut retained);
547    retained.push(thread);
548}
549
550/// Join the retained threads that have returned.
551fn reap(retained: &mut Vec<RegistrationThread>) {
552    let (returned, running): (Vec<_>, Vec<_>) = std::mem::take(retained)
553        .into_iter()
554        .partition(|thread| thread.handle.is_finished());
555    *retained = running;
556    for thread in returned {
557        let _ = thread.handle.join();
558    }
559}
560
561fn sidecar_path(path: &Path, suffix: &str) -> PathBuf {
562    let mut value = path.as_os_str().to_os_string();
563    value.push(suffix);
564    PathBuf::from(value)
565}
566
567/// The database file and its sidecars.
568pub fn sqlite_watch_paths(path: &Path) -> Vec<PathBuf> {
569    vec![
570        path.to_path_buf(),
571        sidecar_path(path, "-wal"),
572        sidecar_path(path, "-shm"),
573    ]
574}
575
576fn sqlite_watch_targets(db_path: &Path) -> Result<(PathBuf, Vec<PathBuf>), String> {
577    let file_name = db_path
578        .file_name()
579        .ok_or_else(|| format!("{} has no database filename", db_path.display()))?;
580    let parent = db_path
581        .parent()
582        .filter(|parent| !parent.as_os_str().is_empty())
583        .unwrap_or_else(|| Path::new("."));
584    let parent = parent.canonicalize().map_err(|error| {
585        format!(
586            "cannot resolve {} for sqlite watch: {error}",
587            parent.display()
588        )
589    })?;
590    // FSEvents reports canonical paths, including /private/var for /var on
591    // macOS. Resolve only the existing directory: database and sidecar files
592    // may not exist yet, or may have been removed when an event is delivered.
593    let watched_paths = sqlite_watch_paths(&parent.join(file_name));
594    Ok((parent, watched_paths))
595}
596
597/// Whether a notification can announce a change to the database: a commit,
598/// a checkpoint, or the database file or its directory replaced or removed.
599///
600/// `watched_paths` are the database's paths as [`sqlite_watch_paths`] names
601/// them in its canonical directory `parent`. A connection's own open and
602/// close are no change: the first connection to open a WAL database creates
603/// its `-wal` and `-shm` files and writes the `-shm` (the WAL index), and the
604/// last to close removes both. A commit appends to the WAL (without one, it
605/// writes the database file), and the WAL index changes only through shared
606/// memory, which no notification reports. So nothing on the `-shm` file is
607/// relevant, nor the creation, removal or metadata of the `-wal`. Every
608/// other event on the database file, the `-wal` or the directory itself is
609/// (an unknown kind included), and so is an event that names no path (a
610/// backend's rescan). An access never is.
611///
612/// On macOS, FSEvents can also report the directory itself when a file in
613/// it is created or removed. The directory stays relevant, since its own
614/// removal or rename matters, so there a connection's own open or close may
615/// still tick.
616pub fn sqlite_watch_event_relevant(
617    event: &notify::Event,
618    parent: &Path,
619    watched_paths: &[PathBuf],
620) -> bool {
621    if matches!(event.kind, notify::EventKind::Access(_)) {
622        return false;
623    }
624    if event.paths.is_empty() {
625        return true;
626    }
627    event.paths.iter().any(|path| {
628        if path == parent {
629            return true;
630        }
631        let Some(watched) = watched_paths.iter().find(|watched| {
632            path == *watched
633                || (path.parent() == watched.parent() && path.file_name() == watched.file_name())
634        }) else {
635            return false;
636        };
637        match WatchedFile::of(watched, watched_paths) {
638            WatchedFile::Database => true,
639            WatchedFile::Wal => !matches!(
640                event.kind,
641                notify::EventKind::Create(_)
642                    | notify::EventKind::Remove(_)
643                    | notify::EventKind::Modify(notify::event::ModifyKind::Metadata(_))
644            ),
645            WatchedFile::WalIndex => false,
646        }
647    })
648}
649
650/// Which of a database's watched files a watched path is.
651#[derive(Debug, Clone, Copy, PartialEq, Eq)]
652enum WatchedFile {
653    Database,
654    /// The `-wal` file, which commits append to.
655    Wal,
656    /// The `-shm` file, the WAL index, which changes through shared memory.
657    WalIndex,
658}
659
660impl WatchedFile {
661    /// `watched`'s place among `watched_paths`: the `-wal` or `-shm` sidecar
662    /// of another watched path, or a database.
663    fn of(watched: &Path, watched_paths: &[PathBuf]) -> Self {
664        for database in watched_paths {
665            if sidecar_path(database, "-wal") == watched {
666                return Self::Wal;
667            }
668            if sidecar_path(database, "-shm") == watched {
669                return Self::WalIndex;
670            }
671        }
672        Self::Database
673    }
674}
675
676/// The directory a watch registers, and the database files whose changes in
677/// it wake the worker.
678struct WatchTargets {
679    parent: PathBuf,
680    watched_paths: Vec<PathBuf>,
681}
682
683/// Creates a watch's notification source: notify's recommended watcher, or a
684/// held fake under the `test-support` feature.
685trait WatchBackend: Send {
686    /// A source that sends on `wake` for each relevant change once it is
687    /// registered.
688    fn create(
689        &self,
690        targets: &WatchTargets,
691        wake: mpsc::Sender<()>,
692    ) -> Result<Box<dyn WatchSource>, String>;
693}
694
695/// A created notification source. Dropping it stops its notifications.
696trait WatchSource: Send {
697    /// Register `directory`. Returns once the platform has registered it,
698    /// which has no bound: macOS FSEvents registration can take tens of
699    /// seconds.
700    fn watch(&mut self, directory: &Path) -> Result<(), String>;
701
702    /// Whether dropping this source can wait on the platform, so a stopped
703    /// watch hands it to the unregister thread ([`unregister`]).
704    fn teardown_waits(&self) -> bool {
705        false
706    }
707}
708
709/// notify's recommended watcher: FSEvents on macOS, inotify on Linux.
710struct NotifyBackend;
711
712impl WatchBackend for NotifyBackend {
713    fn create(
714        &self,
715        targets: &WatchTargets,
716        wake: mpsc::Sender<()>,
717    ) -> Result<Box<dyn WatchSource>, String> {
718        let parent = targets.parent.clone();
719        let watched_paths = targets.watched_paths.clone();
720        let watcher = notify::recommended_watcher(move |result: notify::Result<notify::Event>| {
721            let relevant = matches!(
722                &result,
723                Ok(event) if sqlite_watch_event_relevant(event, &parent, &watched_paths)
724            );
725            #[cfg(any(test, feature = "test-support"))]
726            test_support::log_raw_event(&parent, &result, relevant);
727            match result {
728                Ok(_) if relevant => {
729                    let _ = wake.send(());
730                }
731                Ok(_) => {}
732                Err(error) => {
733                    // The sweep bounds anything a watcher error lost.
734                    tracing::warn!(error = %error, "sqlite change watch reported an error");
735                }
736            }
737        })
738        .map_err(|error| format!("cannot create the sqlite change watcher: {error}"))?;
739        Ok(Box::new(NotifySource(watcher)))
740    }
741}
742
743struct NotifySource(notify::RecommendedWatcher);
744
745impl WatchSource for NotifySource {
746    fn watch(&mut self, directory: &Path) -> Result<(), String> {
747        #[cfg(any(test, feature = "test-support"))]
748        if let Some(delay) = test_support::platform_registration_delay() {
749            thread::sleep(delay);
750        }
751        self.0
752            .watch(directory, RecursiveMode::NonRecursive)
753            .map_err(|error| format!("cannot watch {}: {error}", directory.display()))
754    }
755
756    /// macOS FSEvents: dropping the watcher joins its run-loop thread, which
757    /// makes two round trips to the file-system event service (the current
758    /// event id, then a purge up to it) before it lets go.
759    fn teardown_waits(&self) -> bool {
760        cfg!(target_os = "macos")
761    }
762}
763
764/// The backend that registers a watch of `parent`: notify's, unless test
765/// support holds the registrations in that directory.
766#[cfg_attr(not(any(test, feature = "test-support")), allow(unused_variables))]
767fn watch_backend(parent: &Path) -> Box<dyn WatchBackend> {
768    #[cfg(any(test, feature = "test-support"))]
769    if let Some(held) = test_support::held_backend(parent) {
770        return held;
771    }
772    #[cfg(any(test, feature = "test-support"))]
773    if let Some(observed) = test_support::observed_backend(parent) {
774        return observed;
775    }
776    Box::new(NotifyBackend)
777}
778
779fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
780    mutex.lock().unwrap_or_else(PoisonError::into_inner)
781}
782
783/// Start watching `db_path`. `on_tick` runs on a dedicated thread named
784/// `thread_name`, once per coalesced notification and once per `sweep`
785/// without one, until the watch is stopped or `on_tick` returns
786/// [`SqliteWatchControl::Stop`].
787///
788/// Returns as soon as that worker runs. The notification registration runs
789/// on a thread of its own, `<thread_name>-register`, which ends once the
790/// platform has registered (at once with inotify);
791/// [`SqliteChangeWatch::registration`] reports it. While the platform holds
792/// registrations, each one holds that thread, and on macOS notify's run-loop
793/// thread too, until the platform call returns. At most 64 run at once in a
794/// process: a watch started while 64 are pending runs on its sweep alone
795/// (`Failed`) for the rest of its life and logs a warning. Errors only when
796/// the database's directory cannot be resolved or the worker cannot start.
797pub fn start_sqlite_change_watch(
798    db_path: &Path,
799    thread_name: &str,
800    sweep: Duration,
801    on_tick: impl FnMut(SqliteWatchTick) -> SqliteWatchControl + Send + 'static,
802) -> Result<SqliteChangeWatch, String> {
803    start_with_backend(db_path, thread_name, sweep, on_tick, watch_backend)
804}
805
806fn start_with_backend(
807    db_path: &Path,
808    thread_name: &str,
809    sweep: Duration,
810    mut on_tick: impl FnMut(SqliteWatchTick) -> SqliteWatchControl + Send + 'static,
811    backend: impl FnOnce(&Path) -> Box<dyn WatchBackend>,
812) -> Result<SqliteChangeWatch, String> {
813    let (parent, watched_paths) = sqlite_watch_targets(db_path)?;
814    let backend = backend(&parent);
815    let targets = WatchTargets {
816        parent,
817        watched_paths,
818    };
819    let (wake_tx, wake_rx) = mpsc::channel::<()>();
820    let stop_requested = Arc::new(AtomicBool::new(false));
821    let worker_stop_requested = Arc::clone(&stop_requested);
822
823    // The sweep is the guarantee: its worker starts first, before anything
824    // that can block.
825    let worker = thread::Builder::new()
826        .name(thread_name.to_string())
827        .spawn(move || {
828            let mut recovery_backoff = SQLITE_WATCH_RECOVERY_BACKOFF_MIN;
829            let mut recovery_deadline: Option<Instant> = None;
830            loop {
831                let wait = recovery_deadline
832                    .map(|deadline| deadline.saturating_duration_since(Instant::now()))
833                    .unwrap_or(sweep);
834                let tick = match wake_rx.recv_timeout(wait) {
835                    Ok(()) => SqliteWatchTick::Changed,
836                    Err(mpsc::RecvTimeoutError::Timeout) => SqliteWatchTick::Sweep,
837                    // The watcher (and every sender) is gone: stop.
838                    Err(mpsc::RecvTimeoutError::Disconnected) => break,
839                };
840                if worker_stop_requested.load(Ordering::Acquire) {
841                    break;
842                }
843                if tick == SqliteWatchTick::Changed {
844                    thread::sleep(SQLITE_WATCH_COALESCE);
845                }
846                while wake_rx.try_recv().is_ok() {}
847                // A stop requested while the burst settled still precedes
848                // this tick.
849                if worker_stop_requested.load(Ordering::Acquire) {
850                    break;
851                }
852                if recovery_deadline.is_some_and(|deadline| Instant::now() < deadline) {
853                    // A notification storm must not bypass a failed read's
854                    // recovery deadline.
855                    continue;
856                }
857                match on_tick(tick) {
858                    SqliteWatchControl::Handled => {
859                        recovery_deadline = None;
860                        recovery_backoff = SQLITE_WATCH_RECOVERY_BACKOFF_MIN;
861                    }
862                    SqliteWatchControl::Failed => {
863                        recovery_deadline = Some(Instant::now() + recovery_backoff);
864                        recovery_backoff = recovery_backoff
865                            .saturating_mul(2)
866                            .min(SQLITE_WATCH_RECOVERY_BACKOFF_MAX);
867                    }
868                    SqliteWatchControl::Stop => break,
869                }
870            }
871        })
872        .map_err(|error| format!("cannot start the sqlite change watch thread: {error}"))?;
873
874    let registration = RegistrationCustody::spawn(
875        thread_name,
876        backend,
877        targets,
878        wake_tx.clone(),
879        Arc::clone(&stop_requested),
880    );
881    Ok(SqliteChangeWatch {
882        control: Arc::new(WatchControl {
883            stop_requested,
884            wake: wake_tx,
885        }),
886        registration,
887        worker: WorkerCustody::Running(worker),
888    })
889}
890
891/// The registration thread: create the source and register it, then hand it
892/// to the watch, or discard it when the watch stopped meanwhile.
893fn register(
894    backend: &dyn WatchBackend,
895    targets: &WatchTargets,
896    wake: mpsc::Sender<()>,
897    stop_requested: &AtomicBool,
898    slot: &Mutex<RegistrationSlot>,
899) {
900    if stop_requested.load(Ordering::Acquire) {
901        *lock(slot) = RegistrationSlot::Cancelled;
902        return;
903    }
904    let mut source = match backend.create(targets, wake) {
905        Ok(source) => source,
906        Err(reason) => {
907            settle_error(slot, stop_requested, reason);
908            return;
909        }
910    };
911    // notify 8.2.0's inotify watcher ignores a failed start of its event-loop
912    // thread; registering then panics, and so would its drop, aborting the
913    // process. Contain the panic and leak that source (a sender and a waker)
914    // instead of dropping it. An error, unlike a panic, comes from a live
915    // event loop, so its source drops normally.
916    match std::panic::catch_unwind(AssertUnwindSafe(|| source.watch(&targets.parent))) {
917        Ok(Ok(())) => {}
918        Ok(Err(reason)) => {
919            drop(source);
920            settle_error(slot, stop_requested, reason);
921            return;
922        }
923        Err(_) => {
924            std::mem::forget(source);
925            settle_failed(
926                slot,
927                "registering the notification source panicked: its event loop is not running"
928                    .to_string(),
929            );
930            return;
931        }
932    }
933    {
934        // The stop side sets its flag before it reads the slot, so a flag
935        // read under the slot lock never leaves a source live past a stop.
936        let mut settled = lock(slot);
937        if !stop_requested.load(Ordering::Acquire) {
938            *settled = RegistrationSlot::Registered(Some(source));
939            return;
940        }
941    }
942    // The watch stopped while the platform registered: discard the source
943    // here, so its notifications never start, then settle.
944    drop(source);
945    *lock(slot) = RegistrationSlot::Cancelled;
946}
947
948/// Settle a registration that returned an error: failed, with a warning, while
949/// its watch runs on its sweep alone; cancelled, quietly, once the watch has
950/// stopped, since nothing is left to run on the sweep.
951fn settle_error(slot: &Mutex<RegistrationSlot>, stop_requested: &AtomicBool, reason: String) {
952    if stop_requested.load(Ordering::Acquire) {
953        *lock(slot) = RegistrationSlot::Cancelled;
954    } else {
955        settle_failed(slot, reason);
956    }
957}
958
959fn settle_failed(slot: &Mutex<RegistrationSlot>, reason: String) {
960    tracing::warn!(
961        %reason,
962        "sqlite change watch notifications are unavailable; the watch runs on its sweep alone"
963    );
964    *lock(slot) = RegistrationSlot::Failed(reason);
965}
966
967/// Held watch registrations, for tests of code that starts SQLite change
968/// watches. Every watch started on a database in a held directory blocks in
969/// its notification registration, as a stalled platform registration (macOS
970/// FSEvents) does, until the hold is released.
971#[cfg(any(test, feature = "test-support"))]
972pub mod test_support {
973    use std::collections::HashMap;
974    use std::io::Write as _;
975    use std::path::{Path, PathBuf};
976    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
977    use std::sync::{Arc, Condvar, Mutex, OnceLock, PoisonError, mpsc};
978    use std::time::{Duration, Instant};
979
980    use super::{WatchBackend, WatchSource, WatchTargets, lock};
981
982    /// The variable that slows every platform registration in a test process.
983    pub const PLATFORM_REGISTRATION_DELAY_MS: &str =
984        "MEERKAT_TEST_SQLITE_WATCH_REGISTRATION_DELAY_MS";
985
986    /// How long every platform registration in this process waits before it
987    /// registers: [`PLATFORM_REGISTRATION_DELAY_MS`] milliseconds, read once
988    /// (unset: none). It makes the notify source register as slowly as a
989    /// macOS FSEvents registration can (tens of seconds), on any platform.
990    pub(super) fn platform_registration_delay() -> Option<Duration> {
991        static DELAY: OnceLock<Option<Duration>> = OnceLock::new();
992        *DELAY.get_or_init(|| {
993            std::env::var(PLATFORM_REGISTRATION_DELAY_MS)
994                .ok()
995                .and_then(|millis| millis.parse().ok())
996                .map(Duration::from_millis)
997        })
998    }
999
1000    /// The variable that names a file this test process appends a line to
1001    /// for every notification its platform watchers report, before anything
1002    /// filters it: when it arrived, the watched directory, the event (its
1003    /// kind, paths and attributes) and whether it was relevant. Rows add
1004    /// their own lines ([`log_raw`]). Unset: nothing is logged.
1005    pub const RAW_EVENT_LOG: &str = "MEERKAT_TEST_SQLITE_WATCH_EVENT_LOG";
1006
1007    fn raw_event_log() -> Option<&'static Mutex<std::fs::File>> {
1008        static LOG: OnceLock<Option<Mutex<std::fs::File>>> = OnceLock::new();
1009        LOG.get_or_init(|| {
1010            let path = std::env::var_os(RAW_EVENT_LOG)?;
1011            std::fs::OpenOptions::new()
1012                .create(true)
1013                .append(true)
1014                .open(path)
1015                .ok()
1016                .map(Mutex::new)
1017        })
1018        .as_ref()
1019    }
1020
1021    /// Append `line` to the [`RAW_EVENT_LOG`] file, after the time (seconds
1022    /// and microseconds since the Unix epoch) and this process's id. Without
1023    /// that variable it does nothing.
1024    pub fn log_raw(line: std::fmt::Arguments<'_>) {
1025        let Some(log) = raw_event_log() else {
1026            return;
1027        };
1028        let now = std::time::SystemTime::now()
1029            .duration_since(std::time::UNIX_EPOCH)
1030            .unwrap_or_default();
1031        let _ = writeln!(
1032            lock(log),
1033            "{}.{:06} pid={} {line}",
1034            now.as_secs(),
1035            now.subsec_micros(),
1036            std::process::id()
1037        );
1038    }
1039
1040    /// Log one notification a platform watcher of `parent` reported, and
1041    /// whether the watch took it as relevant.
1042    pub(super) fn log_raw_event(
1043        parent: &Path,
1044        result: &notify::Result<notify::Event>,
1045        relevant: bool,
1046    ) {
1047        if raw_event_log().is_none() {
1048            return;
1049        }
1050        match result {
1051            Ok(event) => log_raw(format_args!(
1052                "event dir={} relevant={relevant} {event:?}",
1053                parent.display()
1054            )),
1055            Err(error) => log_raw(format_args!("error dir={} {error}", parent.display())),
1056        }
1057    }
1058
1059    /// Held directories, by canonical path.
1060    static HOLDS: OnceLock<Mutex<HashMap<PathBuf, Arc<Hold>>>> = OnceLock::new();
1061
1062    fn holds() -> &'static Mutex<HashMap<PathBuf, Arc<Hold>>> {
1063        HOLDS.get_or_init(Mutex::default)
1064    }
1065
1066    #[derive(Default)]
1067    struct Hold {
1068        gate: Mutex<Gate>,
1069        changed: Condvar,
1070        live_sources: AtomicUsize,
1071        dropped_sources: AtomicUsize,
1072    }
1073
1074    #[derive(Default)]
1075    struct Gate {
1076        released: bool,
1077        entered: usize,
1078    }
1079
1080    /// Holds the notification registrations of the watches on databases in
1081    /// one directory. Dropping it releases them and ends the hold.
1082    pub struct HeldRegistrations {
1083        directory: PathBuf,
1084        hold: Arc<Hold>,
1085    }
1086
1087    /// Hold the registration of every watch started from now on for a
1088    /// database in `directory` (an existing directory, matched by its
1089    /// canonical path), until the returned hold is released or dropped. A
1090    /// held registration has created its source and blocks in registering
1091    /// it; once released it registers, without ever notifying.
1092    pub fn hold_registrations_in(directory: &Path) -> std::io::Result<HeldRegistrations> {
1093        let directory = directory.canonicalize()?;
1094        let hold = Arc::new(Hold::default());
1095        let mut holds = lock(holds());
1096        if holds.contains_key(&directory) {
1097            return Err(std::io::Error::new(
1098                std::io::ErrorKind::AlreadyExists,
1099                format!("{} is already held", directory.display()),
1100            ));
1101        }
1102        holds.insert(directory.clone(), Arc::clone(&hold));
1103        Ok(HeldRegistrations { directory, hold })
1104    }
1105
1106    impl HeldRegistrations {
1107        /// Wait, at most `bound`, until `count` registrations have entered
1108        /// the hold. Whether they had.
1109        pub fn wait_entered(&self, count: usize, bound: Duration) -> bool {
1110            let deadline = Instant::now() + bound;
1111            let mut gate = lock(&self.hold.gate);
1112            while gate.entered < count {
1113                let Some(left) = deadline.checked_duration_since(Instant::now()) else {
1114                    return false;
1115                };
1116                gate = self
1117                    .hold
1118                    .changed
1119                    .wait_timeout(gate, left)
1120                    .unwrap_or_else(PoisonError::into_inner)
1121                    .0;
1122            }
1123            true
1124        }
1125
1126        /// Let every held registration register, and every later one in the
1127        /// directory register at once.
1128        pub fn release(&self) {
1129            lock(&self.hold.gate).released = true;
1130            self.hold.changed.notify_all();
1131        }
1132
1133        /// Sources created through this hold and not dropped yet.
1134        pub fn live_sources(&self) -> usize {
1135            self.hold.live_sources.load(Ordering::SeqCst)
1136        }
1137
1138        /// Sources created through this hold and dropped since.
1139        pub fn dropped_sources(&self) -> usize {
1140            self.hold.dropped_sources.load(Ordering::SeqCst)
1141        }
1142    }
1143
1144    impl Drop for HeldRegistrations {
1145        fn drop(&mut self) {
1146            lock(holds()).remove(&self.directory);
1147            self.release();
1148        }
1149    }
1150
1151    /// The held backend for a watch of `parent`, while a hold covers it.
1152    pub(super) fn held_backend(parent: &Path) -> Option<Box<dyn WatchBackend>> {
1153        let hold = lock(holds()).get(parent).cloned()?;
1154        Some(Box::new(HeldBackend(hold)))
1155    }
1156
1157    struct HeldBackend(Arc<Hold>);
1158
1159    impl WatchBackend for HeldBackend {
1160        fn create(
1161            &self,
1162            _targets: &WatchTargets,
1163            wake: mpsc::Sender<()>,
1164        ) -> Result<Box<dyn WatchSource>, String> {
1165            self.0.live_sources.fetch_add(1, Ordering::SeqCst);
1166            Ok(Box::new(HeldSource {
1167                hold: Arc::clone(&self.0),
1168                _wake: wake,
1169            }))
1170        }
1171    }
1172
1173    struct HeldSource {
1174        hold: Arc<Hold>,
1175        /// Kept for the source's life, as a real source keeps its wake.
1176        _wake: mpsc::Sender<()>,
1177    }
1178
1179    impl WatchSource for HeldSource {
1180        fn watch(&mut self, _directory: &Path) -> Result<(), String> {
1181            let mut gate = lock(&self.hold.gate);
1182            gate.entered += 1;
1183            self.hold.changed.notify_all();
1184            while !gate.released {
1185                gate = self
1186                    .hold
1187                    .changed
1188                    .wait(gate)
1189                    .unwrap_or_else(PoisonError::into_inner);
1190            }
1191            Ok(())
1192        }
1193    }
1194
1195    impl Drop for HeldSource {
1196        fn drop(&mut self) {
1197            self.hold.live_sources.fetch_sub(1, Ordering::SeqCst);
1198            self.hold.dropped_sources.fetch_add(1, Ordering::SeqCst);
1199        }
1200    }
1201
1202    /// Observed directories, by canonical path.
1203    static OBSERVATIONS: OnceLock<Mutex<HashMap<PathBuf, Arc<Observation>>>> = OnceLock::new();
1204
1205    fn observations() -> &'static Mutex<HashMap<PathBuf, Arc<Observation>>> {
1206        OBSERVATIONS.get_or_init(Mutex::default)
1207    }
1208
1209    #[derive(Default)]
1210    struct Observation {
1211        registered: Mutex<usize>,
1212        changed: Condvar,
1213        /// Report a database's WAL created or removed also as modified.
1214        wal_lifecycle_as_modified: AtomicBool,
1215        drops: Mutex<SourceDrops>,
1216        drops_changed: Condvar,
1217    }
1218
1219    /// The observed sources' drops, while a test holds them.
1220    #[derive(Default)]
1221    struct SourceDrops {
1222        held: bool,
1223        finished: usize,
1224    }
1225
1226    /// Observes the notification registrations of the watches on databases
1227    /// in one directory: they register with the platform as usual, and this
1228    /// counts the ones that did. Dropping it ends the observation.
1229    pub struct ObservedRegistrations {
1230        directory: PathBuf,
1231        observation: Arc<Observation>,
1232    }
1233
1234    /// Observe the registration of every watch started from now on for a
1235    /// database in `directory` (an existing directory, matched by its
1236    /// canonical path), so a test can order its own file activity after a
1237    /// watch's registration ([`ObservedRegistrations::wait_registered`]).
1238    pub fn observe_registrations_in(directory: &Path) -> std::io::Result<ObservedRegistrations> {
1239        let directory = directory.canonicalize()?;
1240        let observation = Arc::new(Observation::default());
1241        let mut observations = lock(observations());
1242        if observations.contains_key(&directory) {
1243            return Err(std::io::Error::new(
1244                std::io::ErrorKind::AlreadyExists,
1245                format!("{} is already observed", directory.display()),
1246            ));
1247        }
1248        observations.insert(directory.clone(), Arc::clone(&observation));
1249        Ok(ObservedRegistrations {
1250            directory,
1251            observation,
1252        })
1253    }
1254
1255    impl ObservedRegistrations {
1256        /// From now on, the observed watches also report a database's WAL
1257        /// created or removed as modified (`Modify(Any)`), as
1258        /// ReadDirectoryChangesW can report a new or removed file, so a
1259        /// connection's own open and close notify a WAL change. Each report
1260        /// still passes the watch's relevance filter.
1261        pub fn report_wal_lifecycle_as_modified(&self) {
1262            self.observation
1263                .wal_lifecycle_as_modified
1264                .store(true, Ordering::SeqCst);
1265        }
1266
1267        /// From now on, dropping an observed source blocks until
1268        /// [`Self::release_source_drops`], as a macOS FSEvents watcher's drop
1269        /// waits on the file-system event service.
1270        pub fn hold_source_drops(&self) {
1271            lock(&self.observation.drops).held = true;
1272        }
1273
1274        /// Let every held source drop, and every later one at once.
1275        pub fn release_source_drops(&self) {
1276            lock(&self.observation.drops).held = false;
1277            self.observation.drops_changed.notify_all();
1278        }
1279
1280        /// Observed sources dropped so far.
1281        pub fn source_drops_finished(&self) -> usize {
1282            lock(&self.observation.drops).finished
1283        }
1284
1285        /// Wait, at most `bound`, until `count` observed sources have been
1286        /// dropped. Whether they had.
1287        pub fn wait_source_drops_finished(&self, count: usize, bound: Duration) -> bool {
1288            let deadline = Instant::now() + bound;
1289            let mut drops = lock(&self.observation.drops);
1290            while drops.finished < count {
1291                let Some(left) = deadline.checked_duration_since(Instant::now()) else {
1292                    return false;
1293                };
1294                drops = self
1295                    .observation
1296                    .drops_changed
1297                    .wait_timeout(drops, left)
1298                    .unwrap_or_else(PoisonError::into_inner)
1299                    .0;
1300            }
1301            true
1302        }
1303
1304        /// Wait, at most `bound`, until `count` registrations have
1305        /// registered. Whether they had.
1306        pub fn wait_registered(&self, count: usize, bound: Duration) -> bool {
1307            let deadline = Instant::now() + bound;
1308            let mut registered = lock(&self.observation.registered);
1309            while *registered < count {
1310                let Some(left) = deadline.checked_duration_since(Instant::now()) else {
1311                    return false;
1312                };
1313                registered = self
1314                    .observation
1315                    .changed
1316                    .wait_timeout(registered, left)
1317                    .unwrap_or_else(PoisonError::into_inner)
1318                    .0;
1319            }
1320            true
1321        }
1322    }
1323
1324    impl Drop for ObservedRegistrations {
1325        fn drop(&mut self) {
1326            lock(observations()).remove(&self.directory);
1327            self.release_source_drops();
1328        }
1329    }
1330
1331    /// The platform backend, observed, for a watch of `parent` while an
1332    /// observation covers it.
1333    pub(super) fn observed_backend(parent: &Path) -> Option<Box<dyn WatchBackend>> {
1334        let observation = lock(observations()).get(parent).cloned()?;
1335        Some(Box::new(ObservedBackend(observation)))
1336    }
1337
1338    struct ObservedBackend(Arc<Observation>);
1339
1340    impl WatchBackend for ObservedBackend {
1341        fn create(
1342            &self,
1343            targets: &WatchTargets,
1344            wake: mpsc::Sender<()>,
1345        ) -> Result<Box<dyn WatchSource>, String> {
1346            let inner = if self.0.wal_lifecycle_as_modified.load(Ordering::SeqCst) {
1347                wal_lifecycle_as_modified_source(targets, wake)?
1348            } else {
1349                super::NotifyBackend.create(targets, wake)?
1350            };
1351            Ok(Box::new(ObservedSource {
1352                inner: Some(inner),
1353                observation: Arc::clone(&self.0),
1354            }))
1355        }
1356    }
1357
1358    /// The platform watcher, with a database's WAL created or removed also
1359    /// reported as modified; every report passes the relevance filter.
1360    fn wal_lifecycle_as_modified_source(
1361        targets: &WatchTargets,
1362        wake: mpsc::Sender<()>,
1363    ) -> Result<Box<dyn WatchSource>, String> {
1364        use notify::EventKind;
1365        let parent = targets.parent.clone();
1366        let watched_paths = targets.watched_paths.clone();
1367        let wals: Vec<PathBuf> = watched_paths
1368            .iter()
1369            .map(|path| super::sidecar_path(path, "-wal"))
1370            .filter(|wal| watched_paths.contains(wal))
1371            .collect();
1372        let watcher = notify::recommended_watcher(move |result: notify::Result<notify::Event>| {
1373            let Ok(event) = result else {
1374                log_raw_event(&parent, &result, false);
1375                return;
1376            };
1377            let mut reports = vec![event.clone()];
1378            if matches!(event.kind, EventKind::Create(_) | EventKind::Remove(_)) {
1379                for wal in event.paths.iter().filter(|path| wals.contains(path)) {
1380                    reports.push(
1381                        notify::Event::new(EventKind::Modify(notify::event::ModifyKind::Any))
1382                            .add_path(wal.clone()),
1383                    );
1384                }
1385            }
1386            let relevant = reports
1387                .iter()
1388                .any(|report| super::sqlite_watch_event_relevant(report, &parent, &watched_paths));
1389            log_raw_event(&parent, &Ok(event), relevant);
1390            if relevant {
1391                let _ = wake.send(());
1392            }
1393        })
1394        .map_err(|error| format!("cannot create the sqlite change watcher: {error}"))?;
1395        Ok(Box::new(super::NotifySource(watcher)))
1396    }
1397
1398    struct ObservedSource {
1399        /// `None` once dropped.
1400        inner: Option<Box<dyn WatchSource>>,
1401        observation: Arc<Observation>,
1402    }
1403
1404    impl WatchSource for ObservedSource {
1405        fn watch(&mut self, directory: &Path) -> Result<(), String> {
1406            if let Some(inner) = self.inner.as_mut() {
1407                inner.watch(directory)?;
1408            }
1409            *lock(&self.observation.registered) += 1;
1410            self.observation.changed.notify_all();
1411            Ok(())
1412        }
1413
1414        /// The platform's answer, or that a test holds this source's drop.
1415        fn teardown_waits(&self) -> bool {
1416            self.inner
1417                .as_ref()
1418                .is_some_and(|inner| inner.teardown_waits())
1419                || lock(&self.observation.drops).held
1420        }
1421    }
1422
1423    impl Drop for ObservedSource {
1424        fn drop(&mut self) {
1425            let mut drops = lock(&self.observation.drops);
1426            while drops.held {
1427                drops = self
1428                    .observation
1429                    .drops_changed
1430                    .wait(drops)
1431                    .unwrap_or_else(PoisonError::into_inner);
1432            }
1433            drop(drops);
1434            drop(self.inner.take());
1435            lock(&self.observation.drops).finished += 1;
1436            self.observation.drops_changed.notify_all();
1437        }
1438    }
1439}
1440
1441#[cfg(test)]
1442#[allow(clippy::expect_used, clippy::unwrap_used)]
1443mod tests {
1444    use super::*;
1445    use std::sync::atomic::AtomicUsize;
1446
1447    /// Wait, within the registration bound, until the watch's registration
1448    /// ends.
1449    fn settled_registration(watch: &SqliteChangeWatch) -> SqliteWatchRegistration {
1450        let deadline = Instant::now() + REGISTRATION_BOUND;
1451        loop {
1452            let registration = watch.registration();
1453            if registration != SqliteWatchRegistration::Pending || Instant::now() >= deadline {
1454                return registration;
1455            }
1456            thread::sleep(Duration::from_millis(2));
1457        }
1458    }
1459
1460    fn retained_in(directory: &Path) -> Vec<SqliteWatchRetainedRegistration> {
1461        retained_registrations()
1462            .into_iter()
1463            .filter(|retained| retained.directory == directory)
1464            .collect()
1465    }
1466
1467    /// Wait, within the hang guard, for a Changed tick, skipping the sweeps: a
1468    /// sweep that ticked while the registration was pending (a macOS FSEvents
1469    /// registration can outlast the sweep) comes first. A source that never
1470    /// notifies leaves sweeps only, and fails.
1471    fn next_changed(tick_rx: &mpsc::Receiver<SqliteWatchTick>) -> Result<(), String> {
1472        let deadline = Instant::now() + HANG_GUARD;
1473        let mut sweeps = 0_usize;
1474        loop {
1475            match tick_rx.recv_timeout(deadline.saturating_duration_since(Instant::now())) {
1476                Ok(SqliteWatchTick::Changed) => return Ok(()),
1477                Ok(SqliteWatchTick::Sweep) => sweeps += 1,
1478                Err(error) => {
1479                    return Err(format!(
1480                        "no Changed tick within the hang guard, {sweeps} sweeps: {error}"
1481                    ));
1482                }
1483            }
1484        }
1485    }
1486
1487    /// Whether a Changed tick comes within `window`, skipping sweeps.
1488    fn changed_within(tick_rx: &mpsc::Receiver<SqliteWatchTick>, window: Duration) -> bool {
1489        let deadline = Instant::now() + window;
1490        loop {
1491            match tick_rx.recv_timeout(deadline.saturating_duration_since(Instant::now())) {
1492                Ok(SqliteWatchTick::Changed) => return true,
1493                Ok(SqliteWatchTick::Sweep) => {}
1494                Err(_) => return false,
1495            }
1496        }
1497    }
1498
1499    /// Append `bytes` to the existing file at `path` and close it, as a
1500    /// commit appends frames to a WAL that a connection holds open.
1501    fn append(path: &Path, bytes: &[u8]) {
1502        use std::io::Write as _;
1503        let mut file = std::fs::OpenOptions::new().append(true).open(path).unwrap();
1504        file.write_all(bytes).unwrap();
1505        drop(file);
1506        log_row(
1507            path,
1508            format_args!("appended {} bytes and closed", bytes.len()),
1509        );
1510    }
1511
1512    /// Log a row's step for the database file or sidecar `path`, naming its
1513    /// canonical directory as the notifications for it are logged (see
1514    /// [`test_support::RAW_EVENT_LOG`]).
1515    fn log_row(path: &Path, step: std::fmt::Arguments<'_>) {
1516        let dir = path
1517            .parent()
1518            .and_then(|parent| parent.canonicalize().ok())
1519            .unwrap_or_default();
1520        test_support::log_raw(format_args!(
1521            "row dir={} file={:?}: {step}",
1522            dir.display(),
1523            path.file_name().unwrap_or_default()
1524        ));
1525    }
1526
1527    /// A WAL database's WAL exists before the watch starts, as it does while
1528    /// any connection holds the database open, so a write to it is a data
1529    /// change only. (Its creation is a connection's own open, which ticks
1530    /// nothing: see [`sqlite_watch_event_relevant`].)
1531    fn assert_database_write_ticks_changed(db: &Path) {
1532        log_row(db, format_args!("a write row starts on {}", db.display()));
1533        std::fs::write(db, b"").unwrap();
1534        let wal = sidecar_path(db, "-wal");
1535        std::fs::write(&wal, b"").unwrap();
1536        let (tick_tx, tick_rx) = mpsc::channel();
1537        let watch = start_sqlite_change_watch(db, "test-watch", SQLITE_WATCH_SWEEP, move |tick| {
1538            let _ = tick_tx.send(tick);
1539            SqliteWatchControl::Handled
1540        })
1541        .expect("watch starts");
1542        // Notifications are registered after the start returns: a write
1543        // before that is seen by the sweep alone.
1544        assert_eq!(
1545            settled_registration(&watch),
1546            SqliteWatchRegistration::Registered
1547        );
1548        append(&wal, b"frame");
1549        next_changed(&tick_rx).expect("the write is observed within the hang guard");
1550    }
1551
1552    #[test]
1553    fn a_write_to_the_database_ticks_changed() {
1554        let dir = tempfile::tempdir().unwrap();
1555        assert_database_write_ticks_changed(&dir.path().join("store.sqlite3"));
1556    }
1557
1558    #[cfg(unix)]
1559    #[test]
1560    fn symlinked_directory_targets_match_canonical_events_for_missing_files() {
1561        let dir = tempfile::tempdir().unwrap();
1562        let actual = dir.path().join("actual");
1563        let alias = dir.path().join("alias");
1564        std::fs::create_dir(&actual).unwrap();
1565        std::os::unix::fs::symlink(&actual, &alias).unwrap();
1566        let db = alias.join("store.sqlite3");
1567        let (parent, watched_paths) = sqlite_watch_targets(&db).unwrap();
1568        let canonical_parent = actual.canonicalize().unwrap();
1569        assert_eq!(parent, canonical_parent);
1570        assert_eq!(
1571            watched_paths,
1572            sqlite_watch_paths(&canonical_parent.join("store.sqlite3"))
1573        );
1574        assert!(!db.exists());
1575        for path in &watched_paths {
1576            assert!(!path.exists());
1577        }
1578        let [database, wal, _index] = <&[PathBuf; 3]>::try_from(watched_paths.as_slice())
1579            .expect("the database and its two sidecars");
1580        for kind in [
1581            notify::EventKind::Create(notify::event::CreateKind::File),
1582            notify::EventKind::Remove(notify::event::RemoveKind::File),
1583        ] {
1584            let event = notify::Event::new(kind).add_path(database.clone());
1585            assert!(sqlite_watch_event_relevant(&event, &parent, &watched_paths));
1586        }
1587        let append = notify::Event::new(notify::EventKind::Modify(
1588            notify::event::ModifyKind::Data(notify::event::DataChange::Any),
1589        ))
1590        .add_path(wal.clone());
1591        assert!(sqlite_watch_event_relevant(
1592            &append,
1593            &parent,
1594            &watched_paths
1595        ));
1596        for path in [
1597            canonical_parent.join("other.sqlite3-wal"),
1598            canonical_parent.join("other").join("store.sqlite3-wal"),
1599        ] {
1600            let event = notify::Event::new(notify::EventKind::Any).add_path(path);
1601            assert!(!sqlite_watch_event_relevant(
1602                &event,
1603                &parent,
1604                &watched_paths
1605            ));
1606        }
1607        let access = notify::Event::new(notify::EventKind::Access(notify::event::AccessKind::Any))
1608            .add_path(watched_paths[0].clone());
1609        assert!(!sqlite_watch_event_relevant(
1610            &access,
1611            &parent,
1612            &watched_paths
1613        ));
1614    }
1615
1616    /// The `notify` kinds one platform backend reports for a WAL database's
1617    /// files: a connection's own open and close, and a commit.
1618    struct BackendShapes {
1619        /// The first connection to open the database creates its WAL and the
1620        /// WAL index.
1621        created: &'static [notify::EventKind],
1622        /// It writes the index (its truncation and size) and can touch the
1623        /// sidecars' metadata.
1624        written: &'static [notify::EventKind],
1625        /// The last connection to close removes both.
1626        removed: &'static [notify::EventKind],
1627        /// A commit appends to the WAL; without a WAL it writes the database.
1628        appended: &'static [notify::EventKind],
1629    }
1630
1631    /// A connection's own open and close of a WAL database are no change,
1632    /// whichever kinds the backend reports them as; a commit is, and so is
1633    /// any event on the database file or the directory itself.
1634    fn assert_only_a_commit_is_a_change(shapes: &BackendShapes) {
1635        let parent = PathBuf::from("/data");
1636        let database = parent.join("store.sqlite3");
1637        let watched_paths = sqlite_watch_paths(&database);
1638        let wal = sidecar_path(&database, "-wal");
1639        let index = sidecar_path(&database, "-shm");
1640        let relevant = |kind: notify::EventKind, path: &Path| {
1641            sqlite_watch_event_relevant(
1642                &notify::Event::new(kind).add_path(path.to_path_buf()),
1643                &parent,
1644                &watched_paths,
1645            )
1646        };
1647        for &kind in shapes.created.iter().chain(shapes.removed) {
1648            assert!(
1649                !relevant(kind, &wal),
1650                "the WAL created or removed is no change: {kind:?}"
1651            );
1652        }
1653        for &kind in shapes
1654            .created
1655            .iter()
1656            .chain(shapes.written)
1657            .chain(shapes.removed)
1658        {
1659            assert!(
1660                !relevant(kind, &index),
1661                "nothing on the WAL index is a change: {kind:?}"
1662            );
1663        }
1664        for &kind in shapes.appended {
1665            assert!(relevant(kind, &wal), "a WAL append is a change: {kind:?}");
1666            assert!(
1667                relevant(kind, &database),
1668                "a database write is a change: {kind:?}"
1669            );
1670        }
1671        for &kind in shapes.created.iter().chain(shapes.removed) {
1672            assert!(
1673                relevant(kind, &database),
1674                "the database created or removed is a change: {kind:?}"
1675            );
1676            assert!(
1677                relevant(kind, &parent),
1678                "the directory itself is a change: {kind:?}"
1679            );
1680        }
1681        let rescan = notify::Event::new(notify::EventKind::Other);
1682        assert!(
1683            sqlite_watch_event_relevant(&rescan, &parent, &watched_paths),
1684            "a rescan names no path and is a change"
1685        );
1686    }
1687
1688    /// inotify (Linux): IN_CREATE, IN_MODIFY, IN_ATTRIB and IN_DELETE.
1689    #[test]
1690    fn inotify_kinds_of_a_connections_own_open_and_close_are_no_change() {
1691        use notify::EventKind::{Create, Modify, Remove};
1692        use notify::event::{CreateKind, DataChange, MetadataKind, ModifyKind, RemoveKind};
1693        assert_only_a_commit_is_a_change(&BackendShapes {
1694            created: &[Create(CreateKind::File)],
1695            written: &[
1696                Modify(ModifyKind::Data(DataChange::Any)),
1697                Modify(ModifyKind::Metadata(MetadataKind::Any)),
1698            ],
1699            removed: &[Remove(RemoveKind::File)],
1700            appended: &[Modify(ModifyKind::Data(DataChange::Any))],
1701        });
1702    }
1703
1704    /// FSEvents (macOS): one event per flag of a record, so a record that
1705    /// merges the WAL's creation with an append still carries the append's
1706    /// data change.
1707    #[test]
1708    fn fsevents_kinds_of_a_connections_own_open_and_close_are_no_change() {
1709        use notify::EventKind::{Create, Modify, Remove};
1710        use notify::event::{CreateKind, DataChange, MetadataKind, ModifyKind, RemoveKind};
1711        assert_only_a_commit_is_a_change(&BackendShapes {
1712            created: &[Create(CreateKind::File), Create(CreateKind::Any)],
1713            written: &[
1714                Modify(ModifyKind::Data(DataChange::Content)),
1715                Modify(ModifyKind::Metadata(MetadataKind::Any)),
1716                Modify(ModifyKind::Metadata(MetadataKind::Ownership)),
1717                Modify(ModifyKind::Metadata(MetadataKind::Extended)),
1718            ],
1719            removed: &[Remove(RemoveKind::File), Remove(RemoveKind::Any)],
1720            appended: &[Modify(ModifyKind::Data(DataChange::Content))],
1721        });
1722        let parent = PathBuf::from("/data");
1723        let watched_paths = sqlite_watch_paths(&parent.join("store.sqlite3"));
1724        let merged = [
1725            Create(CreateKind::File),
1726            Modify(ModifyKind::Metadata(MetadataKind::Any)),
1727            Modify(ModifyKind::Data(DataChange::Content)),
1728        ];
1729        assert!(
1730            merged.into_iter().any(|kind| sqlite_watch_event_relevant(
1731                &notify::Event::new(kind).add_path(watched_paths[1].clone()),
1732                &parent,
1733                &watched_paths,
1734            )),
1735            "a merged record of the WAL's creation and an append is a change"
1736        );
1737    }
1738
1739    /// ReadDirectoryChangesW (Windows): added, modified and removed, with no
1740    /// finer kinds, so every modification of the WAL stays a change.
1741    #[test]
1742    fn read_directory_changes_kinds_of_a_connections_own_open_and_close_are_no_change() {
1743        use notify::EventKind::{Any, Create, Modify, Remove};
1744        use notify::event::{CreateKind, ModifyKind, RemoveKind};
1745        assert_only_a_commit_is_a_change(&BackendShapes {
1746            created: &[Create(CreateKind::Any)],
1747            written: &[Modify(ModifyKind::Any)],
1748            removed: &[Remove(RemoveKind::Any)],
1749            appended: &[Modify(ModifyKind::Any), Any],
1750        });
1751    }
1752
1753    /// On a WAL database, a connection's own open (the first: it creates the
1754    /// WAL and its index) and close (the last: it removes them) tick no
1755    /// watch; its commit does. inotify reports all of it.
1756    #[cfg(target_os = "linux")]
1757    #[test]
1758    fn a_connections_own_open_and_close_tick_nothing_but_its_commit_does() {
1759        let dir = tempfile::tempdir().unwrap();
1760        let db = wal_database(dir.path());
1761        let wal = sidecar_path(&db, "-wal");
1762        let index = sidecar_path(&db, "-shm");
1763        assert!(
1764            !wal.exists() && !index.exists(),
1765            "the last close removed both"
1766        );
1767        let (tick_tx, tick_rx) = mpsc::channel();
1768        let watch = start_sqlite_change_watch(
1769            &db,
1770            "test-own-open",
1771            Duration::from_secs(3600),
1772            move |tick| {
1773                let _ = tick_tx.send(tick);
1774                SqliteWatchControl::Handled
1775            },
1776        )
1777        .expect("watch starts");
1778        assert_eq!(
1779            settled_registration(&watch),
1780            SqliteWatchRegistration::Registered
1781        );
1782        {
1783            let conn = crate::open(&db, crate::ConnectionProfile::PRIMARY).unwrap();
1784            let rows: i64 = conn
1785                .query_row("SELECT COUNT(*) FROM t", [], |row| row.get(0))
1786                .unwrap();
1787            assert_eq!(rows, 1);
1788            assert!(
1789                wal.exists() && index.exists(),
1790                "the first connection created the WAL and its index"
1791            );
1792        }
1793        assert!(
1794            !wal.exists() && !index.exists(),
1795            "the last connection removed them"
1796        );
1797        // The watch coalesces for 10 ms: a tick of the open or the close
1798        // would have come long before this.
1799        assert_eq!(
1800            tick_rx.recv_timeout(Duration::from_millis(500)),
1801            Err(mpsc::RecvTimeoutError::Timeout),
1802            "a connection's own open and close are no change"
1803        );
1804        {
1805            let conn = crate::open(&db, crate::ConnectionProfile::PRIMARY).unwrap();
1806            conn.execute("INSERT INTO t (v) VALUES (2)", []).unwrap();
1807        }
1808        assert_eq!(
1809            tick_rx
1810                .recv_timeout(HANG_GUARD)
1811                .expect("the commit ticks within the hang guard"),
1812            SqliteWatchTick::Changed
1813        );
1814    }
1815
1816    /// How long a commit row waits for a commit's tick before its connection
1817    /// closes. What it saw is logged (see [`test_support::RAW_EVENT_LOG`]),
1818    /// never required: a platform can report a write only when the file
1819    /// written closes.
1820    const BEFORE_CLOSE_WINDOW: Duration = Duration::from_secs(2);
1821
1822    /// A commit while another connection (a kept reader) holds a WAL database
1823    /// open appends to the existing WAL and ticks the watch. The committing
1824    /// connection closes after its commit, as the stores' connections do.
1825    #[test]
1826    fn a_commit_to_an_existing_wal_ticks_changed() {
1827        let dir = tempfile::tempdir().unwrap();
1828        let db = wal_database(dir.path());
1829        log_row(
1830            &db,
1831            format_args!("a_commit_to_an_existing_wal_ticks_changed starts"),
1832        );
1833        let wal = sidecar_path(&db, "-wal");
1834        let kept = crate::open(&db, crate::ConnectionProfile::PRIMARY).unwrap();
1835        let rows: i64 = kept
1836            .query_row("SELECT COUNT(*) FROM t", [], |row| row.get(0))
1837            .unwrap();
1838        assert_eq!(rows, 1);
1839        assert!(wal.exists(), "the kept connection holds the WAL open");
1840        let (tick_tx, tick_rx) = mpsc::channel();
1841        let watch = start_sqlite_change_watch(
1842            &db,
1843            "test-commit-existing-wal",
1844            Duration::from_secs(3600),
1845            move |tick| {
1846                let _ = tick_tx.send(tick);
1847                SqliteWatchControl::Handled
1848            },
1849        )
1850        .expect("watch starts");
1851        assert_eq!(
1852            settled_registration(&watch),
1853            SqliteWatchRegistration::Registered
1854        );
1855        let writer = crate::open(&db, crate::ConnectionProfile::PRIMARY).unwrap();
1856        writer.execute("INSERT INTO t (v) VALUES (2)", []).unwrap();
1857        let wal_len = std::fs::metadata(&wal).map(|metadata| metadata.len());
1858        log_row(&wal, format_args!("committed, WAL length {wal_len:?}"));
1859        let before_close = changed_within(&tick_rx, BEFORE_CLOSE_WINDOW);
1860        log_row(
1861            &wal,
1862            format_args!("Changed before the writer closed: {before_close}"),
1863        );
1864        drop(writer);
1865        if !before_close {
1866            next_changed(&tick_rx).expect("the commit ticks once its connection closed");
1867            log_row(&wal, format_args!("Changed after the writer closed"));
1868        }
1869        drop(kept);
1870    }
1871
1872    /// A connection that opens a WAL database no other connection holds open
1873    /// creates the WAL, and its commit at once appends to that new WAL: the
1874    /// watch ticks. Whether it ticks before the connection closes (its close,
1875    /// the last, then checkpoints into the database file and removes the
1876    /// WAL) is logged, with every notification (see
1877    /// [`test_support::RAW_EVENT_LOG`]).
1878    #[test]
1879    fn a_commit_that_creates_the_wal_ticks_changed() {
1880        let dir = tempfile::tempdir().unwrap();
1881        let db = wal_database(dir.path());
1882        log_row(
1883            &db,
1884            format_args!("a_commit_that_creates_the_wal_ticks_changed starts"),
1885        );
1886        let wal = sidecar_path(&db, "-wal");
1887        assert!(!wal.exists(), "no connection holds the database open");
1888        let (tick_tx, tick_rx) = mpsc::channel();
1889        let watch = start_sqlite_change_watch(
1890            &db,
1891            "test-commit-creates-wal",
1892            Duration::from_secs(3600),
1893            move |tick| {
1894                let _ = tick_tx.send(tick);
1895                SqliteWatchControl::Handled
1896            },
1897        )
1898        .expect("watch starts");
1899        assert_eq!(
1900            settled_registration(&watch),
1901            SqliteWatchRegistration::Registered
1902        );
1903        let writer = crate::open(&db, crate::ConnectionProfile::PRIMARY).unwrap();
1904        writer.execute("INSERT INTO t (v) VALUES (2)", []).unwrap();
1905        let wal_len = std::fs::metadata(&wal).map(|metadata| metadata.len());
1906        log_row(&wal, format_args!("committed, WAL length {wal_len:?}"));
1907        let before_close = changed_within(&tick_rx, BEFORE_CLOSE_WINDOW);
1908        log_row(
1909            &wal,
1910            format_args!("Changed before the writer closed: {before_close}"),
1911        );
1912        drop(writer);
1913        assert!(!wal.exists(), "the last connection removed the WAL");
1914        if !before_close {
1915            next_changed(&tick_rx).expect("the commit ticks once its connection closed");
1916            log_row(&wal, format_args!("Changed after the writer closed"));
1917        }
1918    }
1919
1920    #[cfg(unix)]
1921    #[test]
1922    fn a_write_through_a_symlinked_directory_ticks_changed() {
1923        let dir = tempfile::tempdir().unwrap();
1924        let actual = dir.path().join("actual");
1925        let alias = dir.path().join("alias");
1926        std::fs::create_dir(&actual).unwrap();
1927        std::os::unix::fs::symlink(&actual, &alias).unwrap();
1928        assert_database_write_ticks_changed(&alias.join("store.sqlite3"));
1929    }
1930
1931    /// The sweep ticks with no notification. Once the watch registers, a
1932    /// platform can report the watched directory's own recent creation (macOS
1933    /// FSEvents does, and an event on the directory itself is relevant), so
1934    /// Changed ticks before the first sweep are skipped.
1935    #[test]
1936    fn the_sweep_ticks_without_a_notification() {
1937        let dir = tempfile::tempdir().unwrap();
1938        let db = dir.path().join("store.sqlite3");
1939        let (tick_tx, tick_rx) = mpsc::channel();
1940        let _watch =
1941            start_sqlite_change_watch(&db, "test-sweep", Duration::from_millis(50), move |tick| {
1942                let _ = tick_tx.send(tick);
1943                SqliteWatchControl::Handled
1944            })
1945            .expect("watch starts");
1946        next_sweep(&tick_rx).expect("the sweep ticks within the hang guard");
1947    }
1948
1949    /// Wait, within the hang guard, for a Sweep tick, skipping Changed ticks.
1950    fn next_sweep(tick_rx: &mpsc::Receiver<SqliteWatchTick>) -> Result<(), String> {
1951        let deadline = Instant::now() + HANG_GUARD;
1952        let mut changed = 0_usize;
1953        loop {
1954            match tick_rx.recv_timeout(deadline.saturating_duration_since(Instant::now())) {
1955                Ok(SqliteWatchTick::Sweep) => return Ok(()),
1956                Ok(SqliteWatchTick::Changed) => changed += 1,
1957                Err(error) => {
1958                    return Err(format!(
1959                        "no Sweep tick within the hang guard, {changed} Changed: {error}"
1960                    ));
1961                }
1962            }
1963        }
1964    }
1965
1966    #[test]
1967    fn stop_ends_the_worker() {
1968        let dir = tempfile::tempdir().unwrap();
1969        let db = dir.path().join("store.sqlite3");
1970        let (tick_tx, tick_rx) = mpsc::channel();
1971        let _watch =
1972            start_sqlite_change_watch(&db, "test-stop", Duration::from_millis(20), move |tick| {
1973                let _ = tick_tx.send(tick);
1974                SqliteWatchControl::Stop
1975            })
1976            .expect("watch starts");
1977        tick_rx
1978            .recv_timeout(Duration::from_secs(10))
1979            .expect("one tick");
1980        assert!(
1981            tick_rx.recv_timeout(Duration::from_millis(500)).is_err(),
1982            "no tick after Stop: the worker ended (its sender is gone)"
1983        );
1984    }
1985
1986    /// A WAL database holding one row, as the stores keep theirs.
1987    fn wal_database(dir: &Path) -> PathBuf {
1988        let db = dir.join("store.sqlite3");
1989        let conn = crate::open(&db, crate::ConnectionProfile::PRIMARY).unwrap();
1990        conn.execute_batch("CREATE TABLE t (v INTEGER); INSERT INTO t (v) VALUES (1);")
1991            .unwrap();
1992        db
1993    }
1994
1995    /// Counts ticks, and reports once when the worker drops its tick closure,
1996    /// which it does as it returns.
1997    struct TickProbe {
1998        ticks: Arc<AtomicUsize>,
1999        returned: mpsc::Sender<()>,
2000    }
2001
2002    impl TickProbe {
2003        fn tick(&self) {
2004            self.ticks.fetch_add(1, Ordering::SeqCst);
2005        }
2006    }
2007
2008    impl Drop for TickProbe {
2009        fn drop(&mut self) {
2010            let _ = self.returned.send(());
2011        }
2012    }
2013
2014    #[test]
2015    fn stop_and_join_waits_for_the_tick_in_flight() {
2016        let dir = tempfile::tempdir().unwrap();
2017        let db = wal_database(dir.path());
2018        let (entered_tx, entered_rx) = mpsc::channel();
2019        let (release_tx, release_rx) = mpsc::channel::<()>();
2020        let finished = Arc::new(AtomicBool::new(false));
2021        let tick_finished = Arc::clone(&finished);
2022        let tick_db = db.clone();
2023        let mut watch = start_sqlite_change_watch(
2024            &db,
2025            "test-held-tick",
2026            Duration::from_millis(20),
2027            move |_tick| {
2028                if tick_finished.load(Ordering::SeqCst) {
2029                    return SqliteWatchControl::Handled;
2030                }
2031                // The held tick keeps a read connection open, as a store
2032                // catch-up does.
2033                let conn = crate::open(&tick_db, crate::ConnectionProfile::ReadOnly).unwrap();
2034                let _ = entered_tx.send(());
2035                let _ = release_rx.recv();
2036                let value: i64 = conn
2037                    .query_row("SELECT v FROM t", [], |row| row.get(0))
2038                    .unwrap();
2039                assert_eq!(value, 1);
2040                drop(conn);
2041                tick_finished.store(true, Ordering::SeqCst);
2042                SqliteWatchControl::Handled
2043            },
2044        )
2045        .expect("watch starts");
2046        entered_rx
2047            .recv_timeout(Duration::from_secs(10))
2048            .expect("a tick is held within the hang guard");
2049
2050        let (stopping_tx, stopping_rx) = mpsc::channel();
2051        let stopper_finished = Arc::clone(&finished);
2052        let stopper = thread::spawn(move || {
2053            let _ = stopping_tx.send(());
2054            let outcome = watch.stop_and_join();
2055            (outcome, stopper_finished.load(Ordering::SeqCst))
2056        });
2057        stopping_rx.recv().unwrap();
2058        release_tx.send(()).unwrap();
2059        let (outcome, finished_at_return) = stopper.join().unwrap();
2060
2061        assert!(
2062            finished_at_return,
2063            "stop_and_join returned while a tick still held its connection"
2064        );
2065        assert_eq!(outcome, SqliteWatchJoin::Joined);
2066        // The joined worker can never open the database again, so nothing
2067        // recreates a file while the directory goes: no sleep, no retry.
2068        std::fs::remove_dir_all(dir.path()).unwrap();
2069        assert!(!dir.path().exists());
2070    }
2071
2072    #[test]
2073    fn stop_and_join_on_the_worker_thread_is_unjoined_and_keeps_custody() {
2074        let dir = tempfile::tempdir().unwrap();
2075        let db = dir.path().join("store.sqlite3");
2076        let custody: Arc<std::sync::Mutex<Option<SqliteChangeWatch>>> = Arc::default();
2077        let tick_custody = Arc::clone(&custody);
2078        let (outcome_tx, outcome_rx) = mpsc::channel();
2079        let watch = start_sqlite_change_watch(
2080            &db,
2081            "test-self-stop",
2082            Duration::from_millis(20),
2083            move |_| {
2084                let outcome = tick_custody
2085                    .lock()
2086                    .unwrap()
2087                    .as_mut()
2088                    .map(SqliteChangeWatch::stop_and_join);
2089                if let Some(outcome) = outcome {
2090                    let _ = outcome_tx.send(outcome);
2091                }
2092                SqliteWatchControl::Handled
2093            },
2094        )
2095        .expect("watch starts");
2096        *custody.lock().unwrap() = Some(watch);
2097
2098        let on_worker = outcome_rx
2099            .recv_timeout(Duration::from_secs(10))
2100            .expect("a tick stops its own watch within the hang guard");
2101        assert_eq!(on_worker, SqliteWatchJoin::UnjoinedOnWorkerThread);
2102
2103        // The self-stop kept the join handle, so from here the worker is
2104        // really joined rather than reported as gone.
2105        let mut watch = custody
2106            .lock()
2107            .unwrap()
2108            .take()
2109            .expect("the watch stays in custody");
2110        assert!(!watch.on_worker_thread());
2111        assert_eq!(watch.stop_and_join(), SqliteWatchJoin::Joined);
2112        assert!(watch.is_finished());
2113        assert_eq!(
2114            watch.stop_and_join(),
2115            SqliteWatchJoin::Joined,
2116            "a repeated stop reports the recorded join"
2117        );
2118    }
2119
2120    #[test]
2121    fn dropping_the_watch_requests_the_stop_without_blocking() {
2122        let dir = tempfile::tempdir().unwrap();
2123        let db = dir.path().join("store.sqlite3");
2124        let (entered_tx, entered_rx) = mpsc::channel();
2125        let (release_tx, release_rx) = mpsc::channel::<()>();
2126        let (returned_tx, returned_rx) = mpsc::channel();
2127        let ticks = Arc::new(AtomicUsize::new(0));
2128        let probe = TickProbe {
2129            ticks: Arc::clone(&ticks),
2130            returned: returned_tx,
2131        };
2132        let watch =
2133            start_sqlite_change_watch(&db, "test-drop", Duration::from_millis(20), move |_| {
2134                probe.tick();
2135                let _ = entered_tx.send(());
2136                let _ = release_rx.recv();
2137                SqliteWatchControl::Handled
2138            })
2139            .expect("watch starts");
2140        entered_rx
2141            .recv_timeout(Duration::from_secs(10))
2142            .expect("a tick is held within the hang guard");
2143
2144        // Dropping never waits for the held tick: this thread releases the
2145        // tick only after the drop returned.
2146        drop(watch);
2147        release_tx.send(()).unwrap();
2148        returned_rx
2149            .recv_timeout(Duration::from_secs(10))
2150            .expect("the worker returns after the held tick");
2151        assert_eq!(
2152            ticks.load(Ordering::SeqCst),
2153            1,
2154            "no tick starts after the stop request"
2155        );
2156    }
2157
2158    /// How long a held registration may keep a start from returning, or the
2159    /// sweep from ticking, before the test calls it held.
2160    const HELD_BOUND: Duration = Duration::from_secs(5);
2161    const HANG_GUARD: Duration = Duration::from_secs(10);
2162    /// How long a row waits for a watch to register with the platform: a hang
2163    /// guard, not a latency bound. A macOS FSEvents registration can take tens
2164    /// of seconds ([`WatchSource::watch`]), so a minute; a registration that
2165    /// never returns still fails the row.
2166    const REGISTRATION_BOUND: Duration = Duration::from_secs(60);
2167    /// How long a row waits for released sources to finish their teardown:
2168    /// a hang guard, not a latency bound. Dropping a macOS FSEvents source
2169    /// joins its run-loop thread, which makes two round trips to the file
2170    /// system event service (the current event id, then a purge up to it)
2171    /// before it lets go ([`WatchSource::teardown_waits`]), and under load two
2172    /// of those teardowns in turn outlasted the 10 s guard. So a minute; a
2173    /// teardown that never ends still fails the row.
2174    const TEARDOWN_BOUND: Duration = Duration::from_secs(60);
2175
2176    #[test]
2177    fn a_held_registration_does_not_hold_the_start() {
2178        let dir = tempfile::tempdir().unwrap();
2179        let held = test_support::hold_registrations_in(dir.path()).unwrap();
2180        let db = dir.path().join("store.sqlite3");
2181        let (started_tx, started_rx) = mpsc::channel();
2182        let starter = thread::spawn(move || {
2183            let watch =
2184                start_sqlite_change_watch(&db, "test-held-start", SQLITE_WATCH_SWEEP, |_| {
2185                    SqliteWatchControl::Handled
2186                });
2187            let _ = started_tx.send(());
2188            watch
2189        });
2190        assert!(
2191            held.wait_entered(1, HANG_GUARD),
2192            "the registration reached its platform call"
2193        );
2194        let started = started_rx.recv_timeout(HELD_BOUND);
2195        held.release();
2196        let watch = starter.join().unwrap().expect("watch starts");
2197        assert_eq!(
2198            started,
2199            Ok(()),
2200            "the start waited for the held notification registration"
2201        );
2202        drop(watch);
2203    }
2204
2205    #[test]
2206    fn the_sweep_ticks_while_the_registration_is_held() {
2207        let dir = tempfile::tempdir().unwrap();
2208        let held = test_support::hold_registrations_in(dir.path()).unwrap();
2209        let db = dir.path().join("store.sqlite3");
2210        let (tick_tx, tick_rx) = mpsc::channel();
2211        let starter = thread::spawn(move || {
2212            start_sqlite_change_watch(
2213                &db,
2214                "test-held-sweep",
2215                Duration::from_millis(20),
2216                move |tick| {
2217                    let _ = tick_tx.send(tick);
2218                    SqliteWatchControl::Handled
2219                },
2220            )
2221        });
2222        assert!(
2223            held.wait_entered(1, HANG_GUARD),
2224            "the registration reached its platform call"
2225        );
2226        let tick = tick_rx.recv_timeout(HELD_BOUND);
2227        held.release();
2228        let watch = starter.join().unwrap().expect("watch starts");
2229        assert_eq!(
2230            tick,
2231            Ok(SqliteWatchTick::Sweep),
2232            "no sweep ran while the notification registration was held"
2233        );
2234        drop(watch);
2235    }
2236
2237    /// A source whose registration fails, as inotify's does past the user's
2238    /// watch limit.
2239    struct FailingBackend;
2240
2241    struct FailingSource;
2242
2243    impl WatchBackend for FailingBackend {
2244        fn create(
2245            &self,
2246            _targets: &WatchTargets,
2247            _wake: mpsc::Sender<()>,
2248        ) -> Result<Box<dyn WatchSource>, String> {
2249            Ok(Box::new(FailingSource))
2250        }
2251    }
2252
2253    impl WatchSource for FailingSource {
2254        fn watch(&mut self, directory: &Path) -> Result<(), String> {
2255            Err(format!(
2256                "cannot watch {}: the watch limit is reached",
2257                directory.display()
2258            ))
2259        }
2260    }
2261
2262    #[test]
2263    fn a_failed_registration_keeps_the_sweep() {
2264        let dir = tempfile::tempdir().unwrap();
2265        let db = dir.path().join("store.sqlite3");
2266        let (tick_tx, tick_rx) = mpsc::channel();
2267        let watch = start_with_backend(
2268            &db,
2269            "test-failed-registration",
2270            Duration::from_millis(20),
2271            move |tick| {
2272                let _ = tick_tx.send(tick);
2273                SqliteWatchControl::Handled
2274            },
2275            |_| Box::new(FailingBackend),
2276        )
2277        .expect("a failed notification registration leaves the watch on its sweep");
2278        assert_eq!(tick_rx.recv_timeout(HANG_GUARD), Ok(SqliteWatchTick::Sweep));
2279        drop(watch);
2280    }
2281
2282    /// Dropping a watch never waits for its notification source's teardown,
2283    /// which on macOS waits on the file-system event service; a watch started
2284    /// at once on the same database registers and sees a write meanwhile, and
2285    /// the stopped watch ticks no more.
2286    #[test]
2287    fn dropping_a_watch_never_waits_for_its_sources_teardown() {
2288        // It holds the process's unregister thread: alone in its process.
2289        alone_in_child_process(
2290            "dropping_a_watch_never_waits_for_its_sources_teardown",
2291            || {
2292                let dir = tempfile::tempdir().unwrap();
2293                let observed = test_support::observe_registrations_in(dir.path()).unwrap();
2294                let db = dir.path().join("store.sqlite3");
2295                log_row(
2296                    &db,
2297                    format_args!("dropping_a_watch_never_waits_for_its_sources_teardown starts"),
2298                );
2299                std::fs::write(&db, b"").unwrap();
2300                // The WAL exists before the watches start: the write below is
2301                // a data change only.
2302                let wal = sidecar_path(&db, "-wal");
2303                std::fs::write(&wal, b"").unwrap();
2304                let first_ticks = Arc::new(AtomicUsize::new(0));
2305                let first =
2306                    start_sqlite_change_watch(&db, "test-teardown-first", SQLITE_WATCH_SWEEP, {
2307                        let first_ticks = Arc::clone(&first_ticks);
2308                        move |_| {
2309                            first_ticks.fetch_add(1, Ordering::SeqCst);
2310                            SqliteWatchControl::Handled
2311                        }
2312                    })
2313                    .expect("watch starts");
2314                assert!(
2315                    observed.wait_registered(1, REGISTRATION_BOUND),
2316                    "the first registered: {:?}",
2317                    first.registration()
2318                );
2319                observed.hold_source_drops();
2320
2321                let (dropped_tx, dropped_rx) = mpsc::channel();
2322                let dropper = thread::spawn(move || {
2323                    drop(first);
2324                    let _ = dropped_tx.send(());
2325                });
2326                let returned = dropped_rx.recv_timeout(HANG_GUARD).is_ok();
2327                if !returned {
2328                    observed.release_source_drops();
2329                }
2330                dropper.join().unwrap();
2331                assert!(
2332                    returned,
2333                    "dropping the watch waited for its source's teardown"
2334                );
2335                let first_ticked = first_ticks.load(Ordering::SeqCst);
2336
2337                // A watch started at once on the same database works while the first
2338                // watch's source is still held.
2339                let (tick_tx, tick_rx) = mpsc::channel();
2340                let second =
2341                    start_sqlite_change_watch(&db, "test-teardown-second", SQLITE_WATCH_SWEEP, {
2342                        move |tick| {
2343                            let _ = tick_tx.send(tick);
2344                            SqliteWatchControl::Handled
2345                        }
2346                    })
2347                    .expect("watch starts");
2348                assert!(
2349                    observed.wait_registered(2, REGISTRATION_BOUND),
2350                    "the second registered: {:?}",
2351                    second.registration()
2352                );
2353                assert_eq!(
2354                    observed.source_drops_finished(),
2355                    0,
2356                    "the first source is held"
2357                );
2358                append(&wal, b"frame");
2359                next_changed(&tick_rx).expect("the second watch sees the write");
2360                // The held source still delivers that write, to the stopped watch
2361                // only, which ticks no more.
2362                assert_eq!(first_ticks.load(Ordering::SeqCst), first_ticked);
2363
2364                drop(second);
2365                observed.release_source_drops();
2366                assert!(
2367                    observed.wait_source_drops_finished(2, TEARDOWN_BOUND),
2368                    "both sources are dropped once released"
2369                );
2370            },
2371        );
2372    }
2373
2374    #[test]
2375    fn stopping_cancels_a_held_registration_and_keeps_its_thread() {
2376        let dir = tempfile::tempdir().unwrap();
2377        let held = test_support::hold_registrations_in(dir.path()).unwrap();
2378        let directory = dir.path().canonicalize().unwrap();
2379        let db = dir.path().join("store.sqlite3");
2380        let mut watch =
2381            start_sqlite_change_watch(&db, "test-held-stop", Duration::from_millis(20), |_| {
2382                SqliteWatchControl::Handled
2383            })
2384            .expect("watch starts");
2385        assert!(
2386            held.wait_entered(1, HANG_GUARD),
2387            "the registration reached its platform call"
2388        );
2389
2390        // The stop joins the worker and never waits for the registration.
2391        assert_eq!(watch.stop_and_join(), SqliteWatchJoin::Joined);
2392        assert_eq!(watch.registration(), SqliteWatchRegistration::Pending);
2393        assert_eq!(held.live_sources(), 1);
2394
2395        // Once the platform call returns, the registration discards its source
2396        // on its own thread instead of handing it to the stopped watch.
2397        held.release();
2398        assert_eq!(
2399            settled_registration(&watch),
2400            SqliteWatchRegistration::Cancelled
2401        );
2402        assert_eq!((held.live_sources(), held.dropped_sources()), (0, 1));
2403        // Its thread stayed with this watch, which joins it now.
2404        assert!(retained_in(&directory).is_empty());
2405        assert_eq!(watch.stop_and_join(), SqliteWatchJoin::Joined);
2406    }
2407
2408    #[test]
2409    fn dropping_a_watch_retains_its_held_registration_until_it_returns() {
2410        let dir = tempfile::tempdir().unwrap();
2411        let held = test_support::hold_registrations_in(dir.path()).unwrap();
2412        let directory = dir.path().canonicalize().unwrap();
2413        let db = dir.path().join("store.sqlite3");
2414        let watch =
2415            start_sqlite_change_watch(&db, "test-held-drop", Duration::from_millis(20), |_| {
2416                SqliteWatchControl::Handled
2417            })
2418            .expect("watch starts");
2419        assert!(
2420            held.wait_entered(1, HANG_GUARD),
2421            "the registration reached its platform call"
2422        );
2423
2424        // Dropping never waits for the registration and never detaches it:
2425        // the process keeps its thread, and reports it.
2426        drop(watch);
2427        let retained = retained_in(&directory);
2428        assert_eq!(retained.len(), 1, "{retained:?}");
2429        assert_eq!(retained[0].thread_name, "test-held-drop-register");
2430        assert_eq!(held.live_sources(), 1);
2431
2432        // Once the platform call returns, the registration discards its
2433        // source, and the process joins its thread.
2434        held.release();
2435        let deadline = Instant::now() + HANG_GUARD;
2436        while !retained_in(&directory).is_empty() && Instant::now() < deadline {
2437            thread::sleep(Duration::from_millis(2));
2438        }
2439        assert!(
2440            retained_in(&directory).is_empty(),
2441            "the process joins a retained registration once it returns"
2442        );
2443        assert_eq!((held.live_sources(), held.dropped_sources()), (0, 1));
2444    }
2445
2446    #[test]
2447    fn a_pending_registration_is_described_as_the_process_retains_it() {
2448        let dir = tempfile::tempdir().unwrap();
2449        let held = test_support::hold_registrations_in(dir.path()).unwrap();
2450        let directory = dir.path().canonicalize().unwrap();
2451        let db = dir.path().join("store.sqlite3");
2452        let watch =
2453            start_sqlite_change_watch(&db, "test-held-pending", Duration::from_millis(20), |_| {
2454                SqliteWatchControl::Handled
2455            })
2456            .expect("watch starts");
2457        assert!(
2458            held.wait_entered(1, HANG_GUARD),
2459            "the registration reached its platform call"
2460        );
2461
2462        // The stop cancels the held registration without ending it: it is
2463        // still pending, and dropping the watch hands exactly it to the
2464        // process.
2465        watch.request_stop();
2466        let pending = watch
2467            .pending_registration()
2468            .expect("a held registration is pending");
2469        assert_eq!(pending.directory, directory);
2470        assert_eq!(pending.thread_name, "test-held-pending-register");
2471        drop(watch);
2472        let retained = retained_in(&directory);
2473        assert_eq!(retained.len(), 1, "{retained:?}");
2474        assert_eq!(
2475            (&retained[0].directory, &retained[0].thread_name),
2476            (&pending.directory, &pending.thread_name)
2477        );
2478
2479        held.release();
2480        let deadline = Instant::now() + HANG_GUARD;
2481        while !retained_in(&directory).is_empty() && Instant::now() < deadline {
2482            thread::sleep(Duration::from_millis(2));
2483        }
2484        assert!(retained_in(&directory).is_empty());
2485    }
2486
2487    #[test]
2488    fn a_settled_registration_is_not_pending() {
2489        let dir = tempfile::tempdir().unwrap();
2490        // Released before the watch starts: the registration completes at
2491        // once, without a platform call.
2492        let held = test_support::hold_registrations_in(dir.path()).unwrap();
2493        held.release();
2494        let db = dir.path().join("store.sqlite3");
2495        let watch =
2496            start_sqlite_change_watch(&db, "test-settled-pending", SQLITE_WATCH_SWEEP, |_| {
2497                SqliteWatchControl::Handled
2498            })
2499            .expect("watch starts");
2500        assert_eq!(
2501            settled_registration(&watch),
2502            SqliteWatchRegistration::Registered
2503        );
2504        assert_eq!(watch.pending_registration(), None);
2505        watch.request_stop();
2506        assert_eq!(watch.pending_registration(), None);
2507    }
2508
2509    #[test]
2510    fn a_failed_registration_reports_its_reason() {
2511        let dir = tempfile::tempdir().unwrap();
2512        let db = dir.path().join("store.sqlite3");
2513        let watch = start_with_backend(
2514            &db,
2515            "test-failed-reason",
2516            SQLITE_WATCH_SWEEP,
2517            |_| SqliteWatchControl::Handled,
2518            |_| Box::new(FailingBackend),
2519        )
2520        .expect("watch starts");
2521        let registration = settled_registration(&watch);
2522        assert!(
2523            matches!(
2524                &registration,
2525                SqliteWatchRegistration::Failed(reason)
2526                    if reason.contains("the watch limit is reached")
2527            ),
2528            "{registration:?}"
2529        );
2530    }
2531
2532    /// Set in the child process of the panicking-registration test.
2533    const PANIC_CHILD: &str = "MEERKAT_TEST_SQLITE_WATCH_PANIC_CHILD";
2534    /// What that child prints once its watch has survived.
2535    const PANIC_CHILD_DONE: &str = "SQLITE_WATCH_PANIC_CONTAINED";
2536
2537    /// A source whose registration panics and whose drop panics again, as
2538    /// notify 8.2.0's inotify watcher does when its event-loop thread never
2539    /// started: dropping it while the first panic unwinds aborts the process.
2540    struct PanickingBackend;
2541
2542    struct PanickingSource;
2543
2544    impl WatchBackend for PanickingBackend {
2545        fn create(
2546            &self,
2547            _targets: &WatchTargets,
2548            _wake: mpsc::Sender<()>,
2549        ) -> Result<Box<dyn WatchSource>, String> {
2550            Ok(Box::new(PanickingSource))
2551        }
2552    }
2553
2554    #[allow(clippy::panic)]
2555    impl WatchSource for PanickingSource {
2556        fn watch(&mut self, _directory: &Path) -> Result<(), String> {
2557            panic!("the notification event loop is gone");
2558        }
2559    }
2560
2561    #[allow(clippy::panic)]
2562    impl Drop for PanickingSource {
2563        fn drop(&mut self) {
2564            panic!("the notification event loop is gone");
2565        }
2566    }
2567
2568    /// A registered source whose teardown waits and whose drop panics.
2569    struct PanicsOnDropBackend;
2570
2571    struct PanicsOnDropSource;
2572
2573    impl WatchBackend for PanicsOnDropBackend {
2574        fn create(
2575            &self,
2576            _targets: &WatchTargets,
2577            _wake: mpsc::Sender<()>,
2578        ) -> Result<Box<dyn WatchSource>, String> {
2579            Ok(Box::new(PanicsOnDropSource))
2580        }
2581    }
2582
2583    impl WatchSource for PanicsOnDropSource {
2584        fn watch(&mut self, _directory: &Path) -> Result<(), String> {
2585            Ok(())
2586        }
2587
2588        fn teardown_waits(&self) -> bool {
2589            true
2590        }
2591    }
2592
2593    #[allow(clippy::panic)]
2594    impl Drop for PanicsOnDropSource {
2595        fn drop(&mut self) {
2596            panic!("the run-loop thread did not shut down");
2597        }
2598    }
2599
2600    /// The unregister thread contains a panicking source drop: a source whose
2601    /// drop panics, queued between two others, leaves both dropped, and the
2602    /// thread still drops a source queued after them.
2603    #[test]
2604    fn a_panicking_source_drop_leaves_the_unregister_thread_running() {
2605        // It holds the process's unregister thread: alone in its process.
2606        alone_in_child_process(
2607            "a_panicking_source_drop_leaves_the_unregister_thread_running",
2608            || {
2609                let dir = tempfile::tempdir().unwrap();
2610                let observed = test_support::observe_registrations_in(dir.path()).unwrap();
2611                let db = dir.path().join("store.sqlite3");
2612                std::fs::write(&db, b"").unwrap();
2613                let observed_watch = |name: &str| {
2614                    start_sqlite_change_watch(&db, name, SQLITE_WATCH_SWEEP, |_| {
2615                        SqliteWatchControl::Handled
2616                    })
2617                    .expect("watch starts")
2618                };
2619                let first = observed_watch("test-unregister-first");
2620                assert!(
2621                    observed.wait_registered(1, REGISTRATION_BOUND),
2622                    "the first registered: {:?}",
2623                    first.registration()
2624                );
2625                let panicking = start_with_backend(
2626                    &dir.path().join("panics.sqlite3"),
2627                    "test-unregister-panics",
2628                    SQLITE_WATCH_SWEEP,
2629                    |_| SqliteWatchControl::Handled,
2630                    |_| Box::new(PanicsOnDropBackend),
2631                )
2632                .expect("watch starts");
2633                assert_eq!(
2634                    settled_registration(&panicking),
2635                    SqliteWatchRegistration::Registered
2636                );
2637                let second = observed_watch("test-unregister-second");
2638                assert!(
2639                    observed.wait_registered(2, REGISTRATION_BOUND),
2640                    "the second registered: {:?}",
2641                    second.registration()
2642                );
2643
2644                // Queued in this order behind the first source's held drop.
2645                observed.hold_source_drops();
2646                drop(first);
2647                drop(panicking);
2648                drop(second);
2649                observed.release_source_drops();
2650                assert!(
2651                    observed.wait_source_drops_finished(2, TEARDOWN_BOUND),
2652                    "both observed sources are dropped around the panicking one"
2653                );
2654
2655                // The thread keeps going: a source queued after the panic goes to it,
2656                // so its held drop does not hold the dropping thread, and it is
2657                // dropped once released.
2658                let third = observed_watch("test-unregister-third");
2659                assert!(
2660                    observed.wait_registered(3, REGISTRATION_BOUND),
2661                    "the third registered: {:?}",
2662                    third.registration()
2663                );
2664                observed.hold_source_drops();
2665                let (dropped_tx, dropped_rx) = mpsc::channel();
2666                let dropper = thread::spawn(move || {
2667                    drop(third);
2668                    let _ = dropped_tx.send(());
2669                });
2670                let returned = dropped_rx.recv_timeout(HANG_GUARD).is_ok();
2671                observed.release_source_drops();
2672                dropper.join().unwrap();
2673                assert!(
2674                    returned,
2675                    "a source queued after the panic went to the unregister thread"
2676                );
2677                assert!(
2678                    observed.wait_source_drops_finished(3, TEARDOWN_BOUND),
2679                    "the unregister thread drops a source queued after the panic"
2680                );
2681                let unregister = UNREGISTER
2682                    .get()
2683                    .and_then(Option::as_ref)
2684                    .expect("the unregister thread started");
2685                assert!(
2686                    !unregister._thread.is_finished(),
2687                    "the unregister thread runs"
2688                );
2689            },
2690        );
2691    }
2692
2693    /// The registration panics in a child process: a panic during a panic
2694    /// aborts the whole process, so only a child can show the watch survives.
2695    #[test]
2696    fn a_panicking_registration_leaves_the_sweep_running() {
2697        if std::env::var_os(PANIC_CHILD).is_some() {
2698            let dir = tempfile::tempdir().unwrap();
2699            let db = dir.path().join("store.sqlite3");
2700            let (tick_tx, tick_rx) = mpsc::channel();
2701            let watch = start_with_backend(
2702                &db,
2703                "test-panicking-registration",
2704                Duration::from_millis(20),
2705                move |tick| {
2706                    let _ = tick_tx.send(tick);
2707                    SqliteWatchControl::Handled
2708                },
2709                |_| Box::new(PanickingBackend),
2710            )
2711            .expect("watch starts");
2712            let registration = settled_registration(&watch);
2713            assert!(
2714                matches!(registration, SqliteWatchRegistration::Failed(_)),
2715                "{registration:?}"
2716            );
2717            assert_eq!(tick_rx.recv_timeout(HANG_GUARD), Ok(SqliteWatchTick::Sweep));
2718            drop(watch);
2719            println!("\n{PANIC_CHILD_DONE}");
2720            return;
2721        }
2722        #[allow(clippy::disallowed_methods)] // the test re-runs its own binary
2723        let output = std::process::Command::new(std::env::current_exe().unwrap())
2724            .args([
2725                "--exact",
2726                "watch::tests::a_panicking_registration_leaves_the_sweep_running",
2727                "--nocapture",
2728                "--test-threads=1",
2729            ])
2730            .env(PANIC_CHILD, "1")
2731            .output()
2732            .unwrap();
2733        let stdout = String::from_utf8_lossy(&output.stdout);
2734        assert!(
2735            output.status.success() && stdout.contains(PANIC_CHILD_DONE),
2736            "the watch did not survive its panicking registration ({}):\n{stdout}\n{}",
2737            output.status,
2738            String::from_utf8_lossy(&output.stderr)
2739        );
2740    }
2741
2742    /// Set in the child process that runs one test of this module alone.
2743    const ALONE_CHILD: &str = "MEERKAT_TEST_SQLITE_WATCH_ALONE_CHILD";
2744    /// What that child prints once the test passed.
2745    const ALONE_CHILD_DONE: &str = "SQLITE_WATCH_ALONE_TEST_PASSED";
2746
2747    /// Run `body`, the body of this module's test `test`, alone in a child
2748    /// process of this test binary, and require it to pass there. The
2749    /// registrations in flight are counted across the process, so a test
2750    /// that takes every place shares its process with no other watch test,
2751    /// whatever runs the tests: nextest's process per test, or plain `cargo
2752    /// test`'s concurrent threads in one process.
2753    fn alone_in_child_process(test: &str, body: impl FnOnce()) {
2754        if std::env::var_os(ALONE_CHILD).is_some() {
2755            body();
2756            println!("\n{ALONE_CHILD_DONE}");
2757            return;
2758        }
2759        #[allow(clippy::disallowed_methods)] // the test re-runs its own binary
2760        let output = std::process::Command::new(std::env::current_exe().unwrap())
2761            .args([
2762                "--exact",
2763                &format!("watch::tests::{test}"),
2764                "--nocapture",
2765                "--test-threads=1",
2766            ])
2767            .env(ALONE_CHILD, "1")
2768            .output()
2769            .unwrap();
2770        let stdout = String::from_utf8_lossy(&output.stdout);
2771        assert!(
2772            output.status.success() && stdout.contains(ALONE_CHILD_DONE),
2773            "{test} failed alone in its child process ({}):\n{stdout}\n{}",
2774            output.status,
2775            String::from_utf8_lossy(&output.stderr)
2776        );
2777    }
2778
2779    #[test]
2780    fn registrations_past_the_cap_leave_new_watches_on_their_sweep() {
2781        alone_in_child_process(
2782            "registrations_past_the_cap_leave_new_watches_on_their_sweep",
2783            || {
2784                // Alone in its process: no other registration holds a place.
2785                assert_eq!(REGISTRATIONS_IN_FLIGHT.load(Ordering::Acquire), 0);
2786                let dir = tempfile::tempdir().unwrap();
2787                let held = test_support::hold_registrations_in(dir.path()).unwrap();
2788                let watches: Vec<_> = (0..MAX_REGISTRATIONS_IN_FLIGHT)
2789                    .map(|index| {
2790                        start_sqlite_change_watch(
2791                            &dir.path().join(format!("store-{index}.sqlite3")),
2792                            "test-cap-held",
2793                            SQLITE_WATCH_SWEEP,
2794                            |_| SqliteWatchControl::Handled,
2795                        )
2796                        .expect("watch starts")
2797                    })
2798                    .collect();
2799                assert!(
2800                    held.wait_entered(MAX_REGISTRATIONS_IN_FLIGHT, HANG_GUARD),
2801                    "every registration under the cap reached its platform call"
2802                );
2803
2804                // One more watch starts with no registration thread, on its sweep
2805                // alone.
2806                let (tick_tx, tick_rx) = mpsc::channel();
2807                let over = start_sqlite_change_watch(
2808                    &dir.path().join("store-over.sqlite3"),
2809                    "test-cap-over",
2810                    Duration::from_millis(20),
2811                    move |tick| {
2812                        let _ = tick_tx.send(tick);
2813                        SqliteWatchControl::Handled
2814                    },
2815                )
2816                .expect("a watch past the cap still starts");
2817                let registration = over.registration();
2818                assert!(
2819                    matches!(
2820                        &registration,
2821                        SqliteWatchRegistration::Failed(reason) if reason.contains("already pending")
2822                    ),
2823                    "{registration:?}"
2824                );
2825                assert!(
2826                    over.registration.thread.is_none(),
2827                    "no registration thread started past the cap"
2828                );
2829                assert_eq!(held.live_sources(), MAX_REGISTRATIONS_IN_FLIGHT);
2830                assert_eq!(tick_rx.recv_timeout(HANG_GUARD), Ok(SqliteWatchTick::Sweep));
2831
2832                // Released, every held registration settles and gives its place back,
2833                // and a fresh watch registers again.
2834                held.release();
2835                for watch in &watches {
2836                    assert_eq!(
2837                        settled_registration(watch),
2838                        SqliteWatchRegistration::Registered
2839                    );
2840                }
2841                let deadline = Instant::now() + HANG_GUARD;
2842                while REGISTRATIONS_IN_FLIGHT.load(Ordering::Acquire) > 0
2843                    && Instant::now() < deadline
2844                {
2845                    thread::sleep(Duration::from_millis(2));
2846                }
2847                let fresh_dir = tempfile::tempdir().unwrap();
2848                let fresh = start_sqlite_change_watch(
2849                    &fresh_dir.path().join("store.sqlite3"),
2850                    "test-cap-fresh",
2851                    SQLITE_WATCH_SWEEP,
2852                    |_| SqliteWatchControl::Handled,
2853                )
2854                .expect("watch starts");
2855                assert_eq!(
2856                    settled_registration(&fresh),
2857                    SqliteWatchRegistration::Registered,
2858                    "a fresh watch registers once every place is given back"
2859                );
2860                drop(watches);
2861                drop(over);
2862            },
2863        );
2864    }
2865
2866    /// A burst of 64 registrations held at once, as a parallel startup holds
2867    /// them while FSEvents stalls, all reach their platform call: none is
2868    /// refused at the cap and left on its sweep for life. The burst is
2869    /// written out, not taken from the cap, so a lower cap fails here.
2870    #[test]
2871    fn a_held_burst_of_64_registrations_all_reach_their_platform_call() {
2872        alone_in_child_process(
2873            "a_held_burst_of_64_registrations_all_reach_their_platform_call",
2874            || {
2875                const BURST: usize = 64;
2876                // Alone in its process: no other registration holds a place.
2877                assert_eq!(REGISTRATIONS_IN_FLIGHT.load(Ordering::Acquire), 0);
2878                let dir = tempfile::tempdir().unwrap();
2879                let held = test_support::hold_registrations_in(dir.path()).unwrap();
2880                let watches: Vec<_> = (0..BURST)
2881                    .map(|index| {
2882                        start_sqlite_change_watch(
2883                            &dir.path().join(format!("store-{index}.sqlite3")),
2884                            "test-burst-held",
2885                            SQLITE_WATCH_SWEEP,
2886                            |_| SqliteWatchControl::Handled,
2887                        )
2888                        .expect("watch starts")
2889                    })
2890                    .collect();
2891                let refused = watches
2892                    .iter()
2893                    .filter(|watch| {
2894                        matches!(watch.registration(), SqliteWatchRegistration::Failed(_))
2895                    })
2896                    .count();
2897                assert_eq!(refused, 0, "registrations of the burst refused at the cap");
2898                assert!(
2899                    held.wait_entered(BURST, HANG_GUARD),
2900                    "every registration of the burst reached its platform call"
2901                );
2902                held.release();
2903                for watch in &watches {
2904                    assert_eq!(
2905                        settled_registration(watch),
2906                        SqliteWatchRegistration::Registered
2907                    );
2908                }
2909            },
2910        );
2911    }
2912
2913    /// Every WARN event this test process emitted, with the name of the
2914    /// thread that emitted it.
2915    static WARNINGS: Mutex<Vec<(Option<String>, String)>> = Mutex::new(Vec::new());
2916
2917    /// A process-wide subscriber that records WARN events into [`WARNINGS`]: a
2918    /// registration settles on its own thread, which a thread-local
2919    /// subscriber would never see.
2920    struct WarningCapture;
2921
2922    struct MessageField(String);
2923
2924    impl tracing::field::Visit for MessageField {
2925        fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
2926            if field.name() == "message" {
2927                self.0 = format!("{value:?}");
2928            }
2929        }
2930    }
2931
2932    impl tracing::Subscriber for WarningCapture {
2933        fn enabled(&self, metadata: &tracing::Metadata<'_>) -> bool {
2934            *metadata.level() == tracing::Level::WARN
2935        }
2936
2937        fn new_span(&self, _span: &tracing::span::Attributes<'_>) -> tracing::span::Id {
2938            tracing::span::Id::from_u64(1)
2939        }
2940
2941        fn record(&self, _span: &tracing::span::Id, _values: &tracing::span::Record<'_>) {}
2942
2943        fn record_follows_from(&self, _span: &tracing::span::Id, _follows: &tracing::span::Id) {}
2944
2945        fn event(&self, event: &tracing::Event<'_>) {
2946            let mut message = MessageField(String::new());
2947            event.record(&mut message);
2948            lock(&WARNINGS).push((thread::current().name().map(str::to_string), message.0));
2949        }
2950
2951        fn enter(&self, _span: &tracing::span::Id) {}
2952
2953        fn exit(&self, _span: &tracing::span::Id) {}
2954    }
2955
2956    /// Install [`WarningCapture`] as this process's global subscriber, once.
2957    fn capture_warnings() {
2958        static INSTALLED: OnceLock<bool> = OnceLock::new();
2959        let installed = *INSTALLED
2960            .get_or_init(|| tracing::subscriber::set_global_default(WarningCapture).is_ok());
2961        assert!(
2962            installed,
2963            "the warning capture is this process's global subscriber"
2964        );
2965    }
2966
2967    fn warnings_from(thread_name: &str) -> Vec<String> {
2968        lock(&WARNINGS)
2969            .iter()
2970            .filter(|(name, _)| name.as_deref() == Some(thread_name))
2971            .map(|(_, message)| message.clone())
2972            .collect()
2973    }
2974
2975    /// A source whose registration waits for its release, then fails, as an
2976    /// inotify watch past the user's limit does.
2977    struct GatedFailingBackend {
2978        entered: mpsc::Sender<()>,
2979        release: Mutex<Option<mpsc::Receiver<()>>>,
2980    }
2981
2982    struct GatedFailingSource {
2983        entered: mpsc::Sender<()>,
2984        release: mpsc::Receiver<()>,
2985    }
2986
2987    impl WatchBackend for GatedFailingBackend {
2988        fn create(
2989            &self,
2990            _targets: &WatchTargets,
2991            _wake: mpsc::Sender<()>,
2992        ) -> Result<Box<dyn WatchSource>, String> {
2993            let release = lock(&self.release)
2994                .take()
2995                .ok_or_else(|| "created twice".to_string())?;
2996            Ok(Box::new(GatedFailingSource {
2997                entered: self.entered.clone(),
2998                release,
2999            }))
3000        }
3001    }
3002
3003    impl WatchSource for GatedFailingSource {
3004        fn watch(&mut self, directory: &Path) -> Result<(), String> {
3005            let _ = self.entered.send(());
3006            let _ = self.release.recv();
3007            Err(format!(
3008                "cannot watch {}: the watch limit is reached",
3009                directory.display()
3010            ))
3011        }
3012    }
3013
3014    /// Start a watch whose registration fails once released. Returns the
3015    /// watch, the signal that its registration entered, and its release.
3016    fn start_failing_after_release(
3017        dir: &Path,
3018        name: &str,
3019    ) -> (SqliteChangeWatch, mpsc::Receiver<()>, mpsc::Sender<()>) {
3020        let (entered_tx, entered_rx) = mpsc::channel();
3021        let (release_tx, release_rx) = mpsc::channel();
3022        let backend = GatedFailingBackend {
3023            entered: entered_tx,
3024            release: Mutex::new(Some(release_rx)),
3025        };
3026        let watch = start_with_backend(
3027            &dir.join("store.sqlite3"),
3028            name,
3029            SQLITE_WATCH_SWEEP,
3030            |_| SqliteWatchControl::Handled,
3031            move |_| Box::new(backend),
3032        )
3033        .expect("watch starts");
3034        (watch, entered_rx, release_tx)
3035    }
3036
3037    #[test]
3038    fn a_registration_failing_after_its_watch_stopped_is_cancelled_quietly() {
3039        capture_warnings();
3040        // A live watch whose registration fails reports it and warns once:
3041        // the capture sees registration threads.
3042        let live_dir = tempfile::tempdir().unwrap();
3043        let (live, live_entered, live_release) =
3044            start_failing_after_release(live_dir.path(), "test-fail-live");
3045        live_entered
3046            .recv_timeout(HANG_GUARD)
3047            .expect("the live registration reached its platform call");
3048        live_release.send(()).unwrap();
3049        assert!(matches!(
3050            settled_registration(&live),
3051            SqliteWatchRegistration::Failed(_)
3052        ));
3053        assert_eq!(warnings_from("test-fail-live-register").len(), 1);
3054
3055        // A watch that stopped first is gone: its failed registration is
3056        // cancelled, with no warning about a sweep nobody runs any more.
3057        let stopped_dir = tempfile::tempdir().unwrap();
3058        let (mut stopped, stopped_entered, stopped_release) =
3059            start_failing_after_release(stopped_dir.path(), "test-fail-stopped");
3060        stopped_entered
3061            .recv_timeout(HANG_GUARD)
3062            .expect("the registration reached its platform call");
3063        assert_eq!(stopped.stop_and_join(), SqliteWatchJoin::Joined);
3064        stopped_release.send(()).unwrap();
3065        assert_eq!(
3066            (
3067                settled_registration(&stopped),
3068                warnings_from("test-fail-stopped-register")
3069            ),
3070            (SqliteWatchRegistration::Cancelled, Vec::<String>::new())
3071        );
3072    }
3073}