Skip to main content

meerkat_sqlite/
watch.rs

1//! Cross-process change watch for one SQLite database file.
2//!
3//! Another process's commit to a database is invisible to this process's
4//! in-memory signals. This watch turns the file-system notifications that can
5//! announce a commit (on the database, its `-wal` sidecar and their
6//! directory) into coalesced ticks for a consumer that then reads durable
7//! state. A connection's own open and close are no change and tick nothing
8//! ([`sqlite_watch_event_relevant`]): otherwise a consumer's own read would
9//! wake it again.
10//!
11//! A notification is a HINT, never a guarantee: SQLite writes WAL frames
12//! before the WAL index makes the commit visible, so a reader woken by an
13//! event can still read the old state, and no later event is promised. The
14//! watch therefore also ticks on a bounded sweep interval; the consumer's
15//! sweep read is the eventual-wake guarantee. Neither the watch nor the sweep
16//! ever defines permission, custody or expiry: correctness rests with the
17//! consumer's durable reads and its own fences.
18//!
19//! Registering for notifications is a hint too, and nothing waits for it. The
20//! operating system can hold a registration without bound (macOS FSEvents
21//! registration has taken over 30 seconds) and nothing can interrupt it, so
22//! the sweep worker starts first and the registration runs on a thread of its
23//! own: starting a watch returns at once, and a registration that fails leaves
24//! the watch on its sweep alone. [`SqliteChangeWatch::registration`] reports
25//! where it stands.
26//!
27//! This is the contract the mob event bus has used since its external watch
28//! landed, extracted so every store shares one implementation.
29//!
30//! Teardown is explicit. Dropping a watch, or [`SqliteWatchStopHandle::request_stop`],
31//! only asks the worker to return and never blocks: a tick already running
32//! finishes on its own thread, unjoined. [`SqliteChangeWatch::stop_and_join`]
33//! is the one operation that waits for the worker, and it cannot do so from
34//! the worker's own thread. Neither waits for a registration that is still
35//! running: a stop cancels it, and its thread discards its result when the
36//! platform call returns. That thread is never detached: it stays with its
37//! watch and, once the watch is dropped, with the process
38//! ([`retained_registrations`]) until it returns.
39
40use std::panic::AssertUnwindSafe;
41use std::path::{Path, PathBuf};
42use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
43use std::sync::{Arc, Mutex, MutexGuard, OnceLock, PoisonError, mpsc};
44use std::thread;
45use std::time::{Duration, Instant};
46
47use notify::{RecursiveMode, Watcher as _};
48
49/// Default sweep interval: the bounded eventual-wake guarantee.
50pub const SQLITE_WATCH_SWEEP: Duration = Duration::from_secs(5);
51/// Settle delay after a notification, so a burst of events (a commit touches
52/// the WAL and its index) coalesces into one tick.
53const SQLITE_WATCH_COALESCE: Duration = Duration::from_millis(10);
54const SQLITE_WATCH_RECOVERY_BACKOFF_MIN: Duration = Duration::from_millis(100);
55const SQLITE_WATCH_RECOVERY_BACKOFF_MAX: Duration = Duration::from_secs(30);
56/// Notification registrations a process runs at once. A stalled platform
57/// (macOS FSEvents) holds each one for as long as it stalls, with two threads
58/// apiece, so a stall holds at most 128 threads; past this many, a new watch
59/// runs on its sweep alone instead of adding two more. A watch refused at the
60/// cap never registers later: it stays on its sweep for the rest of its life,
61/// which for a store held open is the life of the process. That is still
62/// correct, since correctness rests on the sweep and a notification only
63/// wakes the consumer sooner. The cap fits the bursts of a normal startup,
64/// such as parallel store opens. A queue that registers a refused watch once
65/// a place frees was considered and closed: the cliff needs more than 64
66/// registrations in flight at once.
67const MAX_REGISTRATIONS_IN_FLIGHT: usize = 64;
68/// Registration threads running now in this process, for watches alive or
69/// dropped.
70static REGISTRATIONS_IN_FLIGHT: AtomicUsize = AtomicUsize::new(0);
71
72/// Why the consumer is being asked to look.
73#[derive(Debug, Clone, Copy, PartialEq, Eq)]
74pub enum SqliteWatchTick {
75    /// A relevant file-system notification arrived (coalesced).
76    Changed,
77    /// The sweep interval elapsed without a notification.
78    Sweep,
79}
80
81/// What the consumer's tick achieved.
82#[derive(Debug, Clone, Copy, PartialEq, Eq)]
83pub enum SqliteWatchControl {
84    /// The tick was handled.
85    Handled,
86    /// The consumer's durable read failed: ticks are suppressed until a
87    /// bounded exponential backoff elapses, so a notification storm cannot
88    /// hammer a failing store.
89    Failed,
90    /// The consumer is gone: stop the watch.
91    Stop,
92}
93
94/// How [`SqliteChangeWatch::stop_and_join`] ended.
95#[derive(Debug, Clone, Copy, PartialEq, Eq)]
96#[must_use]
97pub enum SqliteWatchJoin {
98    /// The worker returned and was joined: no tick is running and none will
99    /// run.
100    Joined,
101    /// The worker was joined after a tick panicked: no tick is running and
102    /// none will run.
103    JoinedAfterPanic,
104    /// Called on the worker's own thread, from inside a tick. The stop is
105    /// requested, but a thread cannot join itself: the worker returns after
106    /// the current tick, unjoined. The join handle stays in this watch, so
107    /// whoever holds the watch must keep it and join it from another thread.
108    /// Never a settled outcome.
109    UnjoinedOnWorkerThread,
110}
111
112/// Where a watch's notification registration stands
113/// ([`SqliteChangeWatch::registration`]). Notifications only shorten a wake:
114/// the sweep runs whatever this says.
115#[derive(Debug, Clone, PartialEq, Eq)]
116#[non_exhaustive]
117pub enum SqliteWatchRegistration {
118    /// Running on its own thread, which the operating system can hold without
119    /// bound. Until it ends, only the sweep wakes the consumer.
120    Pending,
121    /// Registered. Its notifications stop when the watch does.
122    Registered,
123    /// Failed, or its thread could not start: only the sweep wakes the
124    /// consumer.
125    Failed(String),
126    /// The watch stopped before the registration ended; its thread discarded
127    /// the result.
128    Cancelled,
129}
130
131/// Asks a watch's worker to return, without owning the worker. Cheap to clone;
132/// never blocks and never joins.
133#[derive(Debug, Clone)]
134pub struct SqliteWatchStopHandle {
135    control: Arc<WatchControl>,
136}
137
138impl SqliteWatchStopHandle {
139    /// Ask the worker to return. Once it observes the request it starts no
140    /// further tick; a tick already running finishes first. Never blocks.
141    pub fn request_stop(&self) {
142        self.control.request_stop();
143    }
144}
145
146#[derive(Debug)]
147struct WatchControl {
148    stop_requested: Arc<AtomicBool>,
149    wake: mpsc::Sender<()>,
150}
151
152impl WatchControl {
153    fn request_stop(&self) {
154        self.stop_requested.store(true, Ordering::Release);
155        // Wake a worker waiting for its next tick. A worker that already
156        // returned has dropped the receiver; nothing is left to wake.
157        let _ = self.wake.send(());
158    }
159}
160
161/// Join custody of a watch's worker thread.
162enum WorkerCustody {
163    Running(thread::JoinHandle<()>),
164    Joined(SqliteWatchJoin),
165}
166
167/// A registration's result, shared by its thread and its watch.
168enum RegistrationSlot {
169    Pending,
170    /// Registered; the source until the watch stops.
171    Registered(Option<Box<dyn WatchSource>>),
172    Failed(String),
173    Cancelled,
174}
175
176/// One running registration's place among [`MAX_REGISTRATIONS_IN_FLIGHT`],
177/// given back when its thread ends, or when the thread never starts.
178struct InFlightPlace;
179
180impl InFlightPlace {
181    /// Take a place, or say how many registrations hold them all.
182    fn take() -> Result<Self, usize> {
183        REGISTRATIONS_IN_FLIGHT
184            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |running| {
185                (running < MAX_REGISTRATIONS_IN_FLIGHT).then_some(running + 1)
186            })
187            .map(|_| Self)
188    }
189}
190
191impl Drop for InFlightPlace {
192    fn drop(&mut self) {
193        REGISTRATIONS_IN_FLIGHT.fetch_sub(1, Ordering::AcqRel);
194    }
195}
196
197/// A registration thread and what it registers.
198struct RegistrationThread {
199    handle: thread::JoinHandle<()>,
200    directory: PathBuf,
201    name: String,
202    started: Instant,
203}
204
205impl RegistrationThread {
206    fn describe(&self) -> SqliteWatchRetainedRegistration {
207        SqliteWatchRetainedRegistration {
208            directory: self.directory.clone(),
209            thread_name: self.name.clone(),
210            running_for: self.started.elapsed(),
211        }
212    }
213}
214
215/// Custody of a watch's notification registration: its result, and its
216/// thread until that is joined.
217struct RegistrationCustody {
218    slot: Arc<Mutex<RegistrationSlot>>,
219    /// `None` once joined, or when the thread never started.
220    thread: Option<RegistrationThread>,
221}
222
223impl RegistrationCustody {
224    /// Register `targets` on a thread of its own, named after the watch. A
225    /// thread that cannot start, or a process already running
226    /// [`MAX_REGISTRATIONS_IN_FLIGHT`] registrations, leaves the registration
227    /// failed: the sweep alone wakes the consumer, for the rest of the
228    /// watch's life.
229    fn spawn(
230        watch_name: &str,
231        backend: Box<dyn WatchBackend>,
232        targets: WatchTargets,
233        wake: mpsc::Sender<()>,
234        stop_requested: Arc<AtomicBool>,
235    ) -> Self {
236        let slot = Arc::new(Mutex::new(RegistrationSlot::Pending));
237        let place = match InFlightPlace::take() {
238            Ok(place) => place,
239            Err(running) => {
240                settle_failed(
241                    &slot,
242                    format!(
243                        "{running} sqlite change watch notification registrations are already \
244                         pending (at most {MAX_REGISTRATIONS_IN_FLIGHT})"
245                    ),
246                );
247                return Self { slot, thread: None };
248            }
249        };
250        let thread_slot = Arc::clone(&slot);
251        let directory = targets.parent.clone();
252        let name = format!("{watch_name}-register");
253        let started = Instant::now();
254        let spawned = thread::Builder::new().name(name.clone()).spawn(move || {
255            // Given back as the thread ends, however it ends.
256            let _place = place;
257            register(
258                backend.as_ref(),
259                &targets,
260                wake,
261                &stop_requested,
262                &thread_slot,
263            );
264        });
265        match spawned {
266            Ok(handle) => Self {
267                slot,
268                thread: Some(RegistrationThread {
269                    handle,
270                    directory,
271                    name,
272                    started,
273                }),
274            },
275            Err(error) => {
276                settle_failed(
277                    &slot,
278                    format!("cannot start the sqlite change watch registration thread: {error}"),
279                );
280                Self { slot, thread: None }
281            }
282        }
283    }
284
285    fn state(&self) -> SqliteWatchRegistration {
286        match &*lock(&self.slot) {
287            RegistrationSlot::Pending => SqliteWatchRegistration::Pending,
288            RegistrationSlot::Registered(_) => SqliteWatchRegistration::Registered,
289            RegistrationSlot::Failed(reason) => SqliteWatchRegistration::Failed(reason.clone()),
290            RegistrationSlot::Cancelled => SqliteWatchRegistration::Cancelled,
291        }
292    }
293
294    /// The registration's thread while the registration is pending. Its
295    /// thread settles the slot before it ends, so a pending registration's
296    /// thread is running.
297    fn pending(&self) -> Option<SqliteWatchRetainedRegistration> {
298        if !matches!(*lock(&self.slot), RegistrationSlot::Pending) {
299            return None;
300        }
301        self.thread.as_ref().map(RegistrationThread::describe)
302    }
303
304    /// Stop the notifications, once the stop flag is set: unregister a
305    /// registered source ([`unregister`]), and join the thread if it has
306    /// returned. A registration still running discards its result when it
307    /// ends.
308    fn stop(&mut self) {
309        let source = match &mut *lock(&self.slot) {
310            RegistrationSlot::Registered(source) => source.take(),
311            _ => None,
312        };
313        if let Some(source) = source {
314            unregister(source);
315        }
316        if self
317            .thread
318            .as_ref()
319            .is_some_and(|thread| thread.handle.is_finished())
320            && let Some(thread) = self.thread.take()
321        {
322            let _ = thread.handle.join();
323        }
324    }
325
326    /// The watch is going: stop, and hand a thread still running to the
327    /// process's custody.
328    fn release(&mut self) {
329        self.stop();
330        if let Some(thread) = self.thread.take() {
331            retain_registration(thread);
332        }
333    }
334}
335
336/// A running watch: custody of its worker thread and of its notification
337/// registration.
338///
339/// Dropping it requests the stop and never joins its worker: the worker
340/// returns at its next wake, after any tick that is running, unjoined. Use
341/// [`Self::stop_and_join`] to wait for the worker.
342pub struct SqliteChangeWatch {
343    control: Arc<WatchControl>,
344    registration: RegistrationCustody,
345    worker: WorkerCustody,
346}
347
348impl std::fmt::Debug for SqliteChangeWatch {
349    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
350        f.debug_struct("SqliteChangeWatch").finish_non_exhaustive()
351    }
352}
353
354impl SqliteChangeWatch {
355    /// A handle that can request this watch's stop after the watch itself has
356    /// moved elsewhere.
357    pub fn stop_handle(&self) -> SqliteWatchStopHandle {
358        SqliteWatchStopHandle {
359            control: Arc::clone(&self.control),
360        }
361    }
362
363    /// Ask the worker to return; see [`SqliteWatchStopHandle::request_stop`].
364    pub fn request_stop(&self) {
365        self.control.request_stop();
366    }
367
368    /// Whether the calling thread is this watch's worker, i.e. the caller is
369    /// running inside one of its ticks.
370    pub fn on_worker_thread(&self) -> bool {
371        self.worker_thread_id() == Some(thread::current().id())
372    }
373
374    /// The worker thread's identity until the worker is joined.
375    pub fn worker_thread_id(&self) -> Option<thread::ThreadId> {
376        match &self.worker {
377            WorkerCustody::Running(worker) => Some(worker.thread().id()),
378            WorkerCustody::Joined(_) => None,
379        }
380    }
381
382    /// Whether the worker has returned (joined or not). A returned worker
383    /// runs no tick again.
384    pub fn is_finished(&self) -> bool {
385        match &self.worker {
386            WorkerCustody::Running(worker) => worker.is_finished(),
387            WorkerCustody::Joined(_) => true,
388        }
389    }
390
391    /// Where the notification registration stands. The sweep runs whatever
392    /// it says.
393    pub fn registration(&self) -> SqliteWatchRegistration {
394        self.registration.state()
395    }
396
397    /// The notification registration this watch would leave with the
398    /// process if it were dropped now, described as
399    /// [`retained_registrations`] would list it: `Some` exactly while
400    /// [`Self::registration`] is [`SqliteWatchRegistration::Pending`].
401    ///
402    /// Read after a stop request, it names the registration the stop
403    /// cancelled while the operating system still held it: that thread
404    /// discards the result when the platform call returns, and until then
405    /// stays with this watch or, once the watch is dropped, with the
406    /// process.
407    pub fn pending_registration(&self) -> Option<SqliteWatchRetainedRegistration> {
408        self.registration.pending()
409    }
410
411    /// Request the stop, stop the notifications, and wait for the worker to
412    /// return: a tick that is running finishes first. Unbounded by design: it
413    /// waits exactly as long as that tick runs.
414    ///
415    /// It never waits for a notification registration that is still running:
416    /// the stop cancels it, and its thread stays in this watch until it
417    /// returns ([`Self::registration`] says
418    /// [`SqliteWatchRegistration::Pending`] until then). A later call joins
419    /// that thread once it has.
420    ///
421    /// On the worker's own thread it cannot wait; it returns
422    /// [`SqliteWatchJoin::UnjoinedOnWorkerThread`] and keeps the join handle
423    /// in this watch, so a later call from another thread still joins.
424    /// Calling it again after a join returns the recorded outcome.
425    pub fn stop_and_join(&mut self) -> SqliteWatchJoin {
426        self.control.request_stop();
427        self.registration.stop();
428        if self.on_worker_thread() {
429            return SqliteWatchJoin::UnjoinedOnWorkerThread;
430        }
431        let outcome = match std::mem::replace(
432            &mut self.worker,
433            WorkerCustody::Joined(SqliteWatchJoin::Joined),
434        ) {
435            WorkerCustody::Running(worker) => match worker.join() {
436                Ok(()) => SqliteWatchJoin::Joined,
437                Err(_) => SqliteWatchJoin::JoinedAfterPanic,
438            },
439            WorkerCustody::Joined(outcome) => outcome,
440        };
441        self.worker = WorkerCustody::Joined(outcome);
442        outcome
443    }
444}
445
446impl Drop for SqliteChangeWatch {
447    fn drop(&mut self) {
448        // Never wait here: the last owner can drop a watch on any thread,
449        // including its own worker's. A registration still running goes to
450        // the process's custody.
451        self.control.request_stop();
452        self.registration.release();
453    }
454}
455
456/// Registration threads still running after their watch was dropped.
457static RETAINED_REGISTRATIONS: OnceLock<Mutex<Vec<RegistrationThread>>> = OnceLock::new();
458
459/// The process's unregister thread, `sqlite-watch-unregister`: the queue it
460/// drops sources from, in turn, and the thread itself, which lives as long as
461/// the process. `None` when it could not start.
462static UNREGISTER: OnceLock<Option<Unregister>> = OnceLock::new();
463
464struct Unregister {
465    queue: mpsc::Sender<Box<dyn WatchSource>>,
466    _thread: thread::JoinHandle<()>,
467}
468
469/// Drop a stopped watch's notification source. A source whose drop waits on
470/// the platform ([`WatchSource::teardown_waits`]) goes to the process's one
471/// unregister thread, which drops such sources in turn, so no thread that
472/// stops a watch (a runtime worker shutting down, a watch's own worker) waits
473/// on the platform. However many watches stop at once, that is one thread;
474/// their sources wait in its queue. Until the thread drops it, a source can
475/// still notify, which only wakes its own stopped watch: its worker returns
476/// at that wake without a tick, and a watch started meanwhile on the same
477/// database has its own source and wake. Any other source, or every source
478/// when the thread cannot start, is dropped here.
479fn unregister(source: Box<dyn WatchSource>) {
480    if !source.teardown_waits() {
481        drop(source);
482        return;
483    }
484    let unregister = UNREGISTER.get_or_init(|| {
485        let (queue, sources) = mpsc::channel::<Box<dyn WatchSource>>();
486        thread::Builder::new()
487            .name("sqlite-watch-unregister".to_string())
488            .spawn(move || {
489                for source in sources {
490                    // A drop can panic (notify's macOS watcher expects its
491                    // run-loop thread to join). Contain it, as a registration's
492                    // panic is, so this thread keeps dropping its queue.
493                    if std::panic::catch_unwind(AssertUnwindSafe(move || drop(source))).is_err() {
494                        tracing::warn!(
495                            "dropping a stopped sqlite change watch's notification source panicked"
496                        );
497                    }
498                }
499            })
500            .map(|thread| Unregister {
501                queue,
502                _thread: thread,
503            })
504            .ok()
505    });
506    let source = match unregister {
507        Some(unregister) => match unregister.queue.send(source) {
508            Ok(()) => return,
509            Err(mpsc::SendError(source)) => source,
510        },
511        None => source,
512    };
513    drop(source);
514}
515
516fn retained() -> &'static Mutex<Vec<RegistrationThread>> {
517    RETAINED_REGISTRATIONS.get_or_init(Mutex::default)
518}
519
520/// A notification registration still running after its watch was dropped
521/// ([`retained_registrations`]), or one a live watch would leave with the
522/// process ([`SqliteChangeWatch::pending_registration`]).
523#[derive(Debug, Clone, PartialEq, Eq)]
524#[non_exhaustive]
525pub struct SqliteWatchRetainedRegistration {
526    /// The directory it registers.
527    pub directory: PathBuf,
528    /// The name of its thread.
529    pub thread_name: String,
530    /// How long it has been running.
531    pub running_for: Duration,
532}
533
534/// The notification registrations this process still holds for watches that
535/// are gone. Each one is blocked in the operating system, which nothing can
536/// interrupt, and discards its result when that call returns. Joins the ones
537/// that have returned; never waits.
538pub fn retained_registrations() -> Vec<SqliteWatchRetainedRegistration> {
539    let mut retained = lock(retained());
540    reap(&mut retained);
541    retained.iter().map(RegistrationThread::describe).collect()
542}
543
544fn retain_registration(thread: RegistrationThread) {
545    let mut retained = lock(retained());
546    reap(&mut retained);
547    retained.push(thread);
548}
549
550/// Join the retained threads that have returned.
551fn reap(retained: &mut Vec<RegistrationThread>) {
552    let (returned, running): (Vec<_>, Vec<_>) = std::mem::take(retained)
553        .into_iter()
554        .partition(|thread| thread.handle.is_finished());
555    *retained = running;
556    for thread in returned {
557        let _ = thread.handle.join();
558    }
559}
560
561fn sidecar_path(path: &Path, suffix: &str) -> PathBuf {
562    let mut value = path.as_os_str().to_os_string();
563    value.push(suffix);
564    PathBuf::from(value)
565}
566
567/// The database file and its sidecars.
568pub fn sqlite_watch_paths(path: &Path) -> Vec<PathBuf> {
569    vec![
570        path.to_path_buf(),
571        sidecar_path(path, "-wal"),
572        sidecar_path(path, "-shm"),
573    ]
574}
575
576fn sqlite_watch_targets(db_path: &Path) -> Result<(PathBuf, Vec<PathBuf>), String> {
577    let file_name = db_path
578        .file_name()
579        .ok_or_else(|| format!("{} has no database filename", db_path.display()))?;
580    let parent = db_path
581        .parent()
582        .filter(|parent| !parent.as_os_str().is_empty())
583        .unwrap_or_else(|| Path::new("."));
584    let parent = parent.canonicalize().map_err(|error| {
585        format!(
586            "cannot resolve {} for sqlite watch: {error}",
587            parent.display()
588        )
589    })?;
590    // FSEvents reports canonical paths, including /private/var for /var on
591    // macOS. Resolve only the existing directory: database and sidecar files
592    // may not exist yet, or may have been removed when an event is delivered.
593    let watched_paths = sqlite_watch_paths(&parent.join(file_name));
594    Ok((parent, watched_paths))
595}
596
597/// Whether a notification can announce a change to the database: a commit,
598/// a checkpoint, or the database file or its directory replaced or removed.
599///
600/// `watched_paths` are the database's paths as [`sqlite_watch_paths`] names
601/// them in its canonical directory `parent`. A connection's own open and
602/// close are no change: the first connection to open a WAL database creates
603/// its `-wal` and `-shm` files and writes the `-shm` (the WAL index), and the
604/// last to close removes both. A commit appends to the WAL (without one, it
605/// writes the database file), and the WAL index changes only through shared
606/// memory, which no notification reports. So nothing on the `-shm` file is
607/// relevant, nor the creation, removal or metadata of the `-wal`. Every
608/// other event on the database file, the `-wal` or the directory itself is
609/// (an unknown kind included), and so is an event that names no path (a
610/// backend's rescan). An access never is.
611///
612/// On macOS, FSEvents can also report the directory itself when a file in
613/// it is created or removed. The directory stays relevant, since its own
614/// removal or rename matters, so there a connection's own open or close may
615/// still tick.
616pub fn sqlite_watch_event_relevant(
617    event: &notify::Event,
618    parent: &Path,
619    watched_paths: &[PathBuf],
620) -> bool {
621    if matches!(event.kind, notify::EventKind::Access(_)) {
622        return false;
623    }
624    if event.paths.is_empty() {
625        return true;
626    }
627    event.paths.iter().any(|path| {
628        if path == parent {
629            return true;
630        }
631        let Some(watched) = watched_paths.iter().find(|watched| {
632            path == *watched
633                || (path.parent() == watched.parent() && path.file_name() == watched.file_name())
634        }) else {
635            return false;
636        };
637        match WatchedFile::of(watched, watched_paths) {
638            WatchedFile::Database => true,
639            WatchedFile::Wal => !matches!(
640                event.kind,
641                notify::EventKind::Create(_)
642                    | notify::EventKind::Remove(_)
643                    | notify::EventKind::Modify(notify::event::ModifyKind::Metadata(_))
644            ),
645            WatchedFile::WalIndex => false,
646        }
647    })
648}
649
650/// Which of a database's watched files a watched path is.
651#[derive(Debug, Clone, Copy, PartialEq, Eq)]
652enum WatchedFile {
653    Database,
654    /// The `-wal` file, which commits append to.
655    Wal,
656    /// The `-shm` file, the WAL index, which changes through shared memory.
657    WalIndex,
658}
659
660impl WatchedFile {
661    /// `watched`'s place among `watched_paths`: the `-wal` or `-shm` sidecar
662    /// of another watched path, or a database.
663    fn of(watched: &Path, watched_paths: &[PathBuf]) -> Self {
664        for database in watched_paths {
665            if sidecar_path(database, "-wal") == watched {
666                return Self::Wal;
667            }
668            if sidecar_path(database, "-shm") == watched {
669                return Self::WalIndex;
670            }
671        }
672        Self::Database
673    }
674}
675
676/// The directory a watch registers, and the database files whose changes in
677/// it wake the worker.
678struct WatchTargets {
679    parent: PathBuf,
680    watched_paths: Vec<PathBuf>,
681}
682
683/// Creates a watch's notification source: notify's recommended watcher, or a
684/// held fake under the `test-support` feature.
685trait WatchBackend: Send {
686    /// A source that sends on `wake` for each relevant change once it is
687    /// registered.
688    fn create(
689        &self,
690        targets: &WatchTargets,
691        wake: mpsc::Sender<()>,
692    ) -> Result<Box<dyn WatchSource>, String>;
693}
694
695/// A created notification source. Dropping it stops its notifications.
696trait WatchSource: Send {
697    /// Register `directory`. Returns once the platform has registered it,
698    /// which has no bound: macOS FSEvents registration can take tens of
699    /// seconds.
700    fn watch(&mut self, directory: &Path) -> Result<(), String>;
701
702    /// Whether dropping this source can wait on the platform, so a stopped
703    /// watch hands it to the unregister thread ([`unregister`]).
704    fn teardown_waits(&self) -> bool {
705        false
706    }
707}
708
709/// notify's recommended watcher: FSEvents on macOS, inotify on Linux.
710struct NotifyBackend;
711
712impl WatchBackend for NotifyBackend {
713    fn create(
714        &self,
715        targets: &WatchTargets,
716        wake: mpsc::Sender<()>,
717    ) -> Result<Box<dyn WatchSource>, String> {
718        let parent = targets.parent.clone();
719        let watched_paths = targets.watched_paths.clone();
720        let watcher = notify::recommended_watcher(move |result: notify::Result<notify::Event>| {
721            let relevant = matches!(
722                &result,
723                Ok(event) if sqlite_watch_event_relevant(event, &parent, &watched_paths)
724            );
725            #[cfg(any(test, feature = "test-support"))]
726            test_support::log_raw_event(&parent, &result, relevant);
727            match result {
728                Ok(_) if relevant => {
729                    let _ = wake.send(());
730                }
731                Ok(_) => {}
732                Err(error) => {
733                    // The sweep bounds anything a watcher error lost.
734                    tracing::warn!(error = %error, "sqlite change watch reported an error");
735                }
736            }
737        })
738        .map_err(|error| format!("cannot create the sqlite change watcher: {error}"))?;
739        Ok(Box::new(NotifySource(watcher)))
740    }
741}
742
743struct NotifySource(notify::RecommendedWatcher);
744
745impl WatchSource for NotifySource {
746    fn watch(&mut self, directory: &Path) -> Result<(), String> {
747        #[cfg(any(test, feature = "test-support"))]
748        if let Some(delay) = test_support::platform_registration_delay() {
749            thread::sleep(delay);
750        }
751        self.0
752            .watch(directory, RecursiveMode::NonRecursive)
753            .map_err(|error| format!("cannot watch {}: {error}", directory.display()))
754    }
755
756    /// macOS FSEvents: dropping the watcher joins its run-loop thread, which
757    /// makes two round trips to the file-system event service (the current
758    /// event id, then a purge up to it) before it lets go.
759    fn teardown_waits(&self) -> bool {
760        cfg!(target_os = "macos")
761    }
762}
763
764/// The backend that registers a watch of `parent`: notify's, unless test
765/// support holds the registrations in that directory.
766#[cfg_attr(not(any(test, feature = "test-support")), allow(unused_variables))]
767fn watch_backend(parent: &Path) -> Box<dyn WatchBackend> {
768    #[cfg(any(test, feature = "test-support"))]
769    if let Some(held) = test_support::held_backend(parent) {
770        return held;
771    }
772    #[cfg(any(test, feature = "test-support"))]
773    if let Some(observed) = test_support::observed_backend(parent) {
774        return observed;
775    }
776    Box::new(NotifyBackend)
777}
778
779fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
780    mutex.lock().unwrap_or_else(PoisonError::into_inner)
781}
782
783/// Start watching `db_path`. `on_tick` runs on a dedicated thread named
784/// `thread_name`, once per coalesced notification and once per `sweep`
785/// without one, until the watch is stopped or `on_tick` returns
786/// [`SqliteWatchControl::Stop`].
787///
788/// Returns as soon as that worker runs. The notification registration runs
789/// on a thread of its own, `<thread_name>-register`, which ends once the
790/// platform has registered (at once with inotify);
791/// [`SqliteChangeWatch::registration`] reports it. While the platform holds
792/// registrations, each one holds that thread, and on macOS notify's run-loop
793/// thread too, until the platform call returns. At most 64 run at once in a
794/// process: a watch started while 64 are pending runs on its sweep alone
795/// (`Failed`) for the rest of its life and logs a warning. Errors only when
796/// the database's directory cannot be resolved or the worker cannot start.
797pub fn start_sqlite_change_watch(
798    db_path: &Path,
799    thread_name: &str,
800    sweep: Duration,
801    on_tick: impl FnMut(SqliteWatchTick) -> SqliteWatchControl + Send + 'static,
802) -> Result<SqliteChangeWatch, String> {
803    start_with_backend(db_path, thread_name, sweep, on_tick, watch_backend)
804}
805
806fn start_with_backend(
807    db_path: &Path,
808    thread_name: &str,
809    sweep: Duration,
810    mut on_tick: impl FnMut(SqliteWatchTick) -> SqliteWatchControl + Send + 'static,
811    backend: impl FnOnce(&Path) -> Box<dyn WatchBackend>,
812) -> Result<SqliteChangeWatch, String> {
813    let (parent, watched_paths) = sqlite_watch_targets(db_path)?;
814    let backend = backend(&parent);
815    let targets = WatchTargets {
816        parent,
817        watched_paths,
818    };
819    let (wake_tx, wake_rx) = mpsc::channel::<()>();
820    let stop_requested = Arc::new(AtomicBool::new(false));
821    let worker_stop_requested = Arc::clone(&stop_requested);
822
823    // The sweep is the guarantee: its worker starts first, before anything
824    // that can block.
825    let worker = thread::Builder::new()
826        .name(thread_name.to_string())
827        .spawn(move || {
828            let mut recovery_backoff = SQLITE_WATCH_RECOVERY_BACKOFF_MIN;
829            let mut recovery_deadline: Option<Instant> = None;
830            loop {
831                let wait = recovery_deadline
832                    .map(|deadline| deadline.saturating_duration_since(Instant::now()))
833                    .unwrap_or(sweep);
834                let tick = match wake_rx.recv_timeout(wait) {
835                    Ok(()) => SqliteWatchTick::Changed,
836                    Err(mpsc::RecvTimeoutError::Timeout) => SqliteWatchTick::Sweep,
837                    // The watcher (and every sender) is gone: stop.
838                    Err(mpsc::RecvTimeoutError::Disconnected) => break,
839                };
840                if worker_stop_requested.load(Ordering::Acquire) {
841                    break;
842                }
843                if tick == SqliteWatchTick::Changed {
844                    thread::sleep(SQLITE_WATCH_COALESCE);
845                }
846                while wake_rx.try_recv().is_ok() {}
847                // A stop requested while the burst settled still precedes
848                // this tick.
849                if worker_stop_requested.load(Ordering::Acquire) {
850                    break;
851                }
852                if recovery_deadline.is_some_and(|deadline| Instant::now() < deadline) {
853                    // A notification storm must not bypass a failed read's
854                    // recovery deadline.
855                    continue;
856                }
857                match on_tick(tick) {
858                    SqliteWatchControl::Handled => {
859                        recovery_deadline = None;
860                        recovery_backoff = SQLITE_WATCH_RECOVERY_BACKOFF_MIN;
861                    }
862                    SqliteWatchControl::Failed => {
863                        recovery_deadline = Some(Instant::now() + recovery_backoff);
864                        recovery_backoff = recovery_backoff
865                            .saturating_mul(2)
866                            .min(SQLITE_WATCH_RECOVERY_BACKOFF_MAX);
867                    }
868                    SqliteWatchControl::Stop => break,
869                }
870            }
871        })
872        .map_err(|error| format!("cannot start the sqlite change watch thread: {error}"))?;
873
874    let registration = RegistrationCustody::spawn(
875        thread_name,
876        backend,
877        targets,
878        wake_tx.clone(),
879        Arc::clone(&stop_requested),
880    );
881    Ok(SqliteChangeWatch {
882        control: Arc::new(WatchControl {
883            stop_requested,
884            wake: wake_tx,
885        }),
886        registration,
887        worker: WorkerCustody::Running(worker),
888    })
889}
890
891/// The registration thread: create the source and register it, then hand it
892/// to the watch, or discard it when the watch stopped meanwhile.
893fn register(
894    backend: &dyn WatchBackend,
895    targets: &WatchTargets,
896    wake: mpsc::Sender<()>,
897    stop_requested: &AtomicBool,
898    slot: &Mutex<RegistrationSlot>,
899) {
900    if stop_requested.load(Ordering::Acquire) {
901        *lock(slot) = RegistrationSlot::Cancelled;
902        return;
903    }
904    let mut source = match backend.create(targets, wake) {
905        Ok(source) => source,
906        Err(reason) => {
907            settle_error(slot, stop_requested, reason);
908            return;
909        }
910    };
911    // notify 8.2.0's inotify watcher ignores a failed start of its event-loop
912    // thread; registering then panics, and so would its drop, aborting the
913    // process. Contain the panic and leak that source (a sender and a waker)
914    // instead of dropping it. An error, unlike a panic, comes from a live
915    // event loop, so its source drops normally.
916    match std::panic::catch_unwind(AssertUnwindSafe(|| source.watch(&targets.parent))) {
917        Ok(Ok(())) => {}
918        Ok(Err(reason)) => {
919            drop(source);
920            settle_error(slot, stop_requested, reason);
921            return;
922        }
923        Err(_) => {
924            std::mem::forget(source);
925            settle_failed(
926                slot,
927                "registering the notification source panicked: its event loop is not running"
928                    .to_string(),
929            );
930            return;
931        }
932    }
933    {
934        // The stop side sets its flag before it reads the slot, so a flag
935        // read under the slot lock never leaves a source live past a stop.
936        let mut settled = lock(slot);
937        if !stop_requested.load(Ordering::Acquire) {
938            *settled = RegistrationSlot::Registered(Some(source));
939            return;
940        }
941    }
942    // The watch stopped while the platform registered: discard the source
943    // here, so its notifications never start, then settle.
944    drop(source);
945    *lock(slot) = RegistrationSlot::Cancelled;
946}
947
948/// Settle a registration that returned an error: failed, with a warning, while
949/// its watch runs on its sweep alone; cancelled, quietly, once the watch has
950/// stopped, since nothing is left to run on the sweep.
951fn settle_error(slot: &Mutex<RegistrationSlot>, stop_requested: &AtomicBool, reason: String) {
952    if stop_requested.load(Ordering::Acquire) {
953        *lock(slot) = RegistrationSlot::Cancelled;
954    } else {
955        settle_failed(slot, reason);
956    }
957}
958
959fn settle_failed(slot: &Mutex<RegistrationSlot>, reason: String) {
960    tracing::warn!(
961        %reason,
962        "sqlite change watch notifications are unavailable; the watch runs on its sweep alone"
963    );
964    *lock(slot) = RegistrationSlot::Failed(reason);
965}
966
967/// Held watch registrations, for tests of code that starts SQLite change
968/// watches. Every watch started on a database in a held directory blocks in
969/// its notification registration, as a stalled platform registration (macOS
970/// FSEvents) does, until the hold is released.
971#[cfg(any(test, feature = "test-support"))]
972pub mod test_support {
973    use std::collections::HashMap;
974    use std::io::Write as _;
975    use std::path::{Path, PathBuf};
976    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
977    use std::sync::{Arc, Condvar, Mutex, OnceLock, PoisonError, mpsc};
978    use std::time::{Duration, Instant};
979
980    use super::{WatchBackend, WatchSource, WatchTargets, lock};
981
982    /// The variable that slows every platform registration in a test process.
983    pub const PLATFORM_REGISTRATION_DELAY_MS: &str =
984        "MEERKAT_TEST_SQLITE_WATCH_REGISTRATION_DELAY_MS";
985
986    /// How long every platform registration in this process waits before it
987    /// registers: [`PLATFORM_REGISTRATION_DELAY_MS`] milliseconds, read once
988    /// (unset: none). It makes the notify source register as slowly as a
989    /// macOS FSEvents registration can (tens of seconds), on any platform.
990    pub(super) fn platform_registration_delay() -> Option<Duration> {
991        static DELAY: OnceLock<Option<Duration>> = OnceLock::new();
992        *DELAY.get_or_init(|| {
993            std::env::var(PLATFORM_REGISTRATION_DELAY_MS)
994                .ok()
995                .and_then(|millis| millis.parse().ok())
996                .map(Duration::from_millis)
997        })
998    }
999
1000    /// The variable that names a file this test process appends a line to
1001    /// for every notification its platform watchers report, before anything
1002    /// filters it: when it arrived, the watched directory, the event (its
1003    /// kind, paths and attributes) and whether it was relevant. Rows add
1004    /// their own lines ([`log_raw`]). Unset: nothing is logged.
1005    pub const RAW_EVENT_LOG: &str = "MEERKAT_TEST_SQLITE_WATCH_EVENT_LOG";
1006
1007    fn raw_event_log() -> Option<&'static Mutex<std::fs::File>> {
1008        static LOG: OnceLock<Option<Mutex<std::fs::File>>> = OnceLock::new();
1009        LOG.get_or_init(|| {
1010            let path = std::env::var_os(RAW_EVENT_LOG)?;
1011            std::fs::OpenOptions::new()
1012                .create(true)
1013                .append(true)
1014                .open(path)
1015                .ok()
1016                .map(Mutex::new)
1017        })
1018        .as_ref()
1019    }
1020
1021    /// Append `line` to the [`RAW_EVENT_LOG`] file, after the time (seconds
1022    /// and microseconds since the Unix epoch) and this process's id. Without
1023    /// that variable it does nothing.
1024    pub fn log_raw(line: std::fmt::Arguments<'_>) {
1025        let Some(log) = raw_event_log() else {
1026            return;
1027        };
1028        let now = std::time::SystemTime::now()
1029            .duration_since(std::time::UNIX_EPOCH)
1030            .unwrap_or_default();
1031        let _ = writeln!(
1032            lock(log),
1033            "{}.{:06} pid={} {line}",
1034            now.as_secs(),
1035            now.subsec_micros(),
1036            std::process::id()
1037        );
1038    }
1039
1040    /// Log one notification a platform watcher of `parent` reported, and
1041    /// whether the watch took it as relevant.
1042    pub(super) fn log_raw_event(
1043        parent: &Path,
1044        result: &notify::Result<notify::Event>,
1045        relevant: bool,
1046    ) {
1047        if raw_event_log().is_none() {
1048            return;
1049        }
1050        match result {
1051            Ok(event) => log_raw(format_args!(
1052                "event dir={} relevant={relevant} {event:?}",
1053                parent.display()
1054            )),
1055            Err(error) => log_raw(format_args!("error dir={} {error}", parent.display())),
1056        }
1057    }
1058
1059    /// Held directories, by canonical path.
1060    static HOLDS: OnceLock<Mutex<HashMap<PathBuf, Arc<Hold>>>> = OnceLock::new();
1061
1062    fn holds() -> &'static Mutex<HashMap<PathBuf, Arc<Hold>>> {
1063        HOLDS.get_or_init(Mutex::default)
1064    }
1065
1066    #[derive(Default)]
1067    struct Hold {
1068        gate: Mutex<Gate>,
1069        changed: Condvar,
1070        live_sources: AtomicUsize,
1071        dropped_sources: AtomicUsize,
1072    }
1073
1074    #[derive(Default)]
1075    struct Gate {
1076        released: bool,
1077        entered: usize,
1078    }
1079
1080    /// Holds the notification registrations of the watches on databases in
1081    /// one directory. Dropping it releases them and ends the hold.
1082    pub struct HeldRegistrations {
1083        directory: PathBuf,
1084        hold: Arc<Hold>,
1085    }
1086
1087    /// Hold the registration of every watch started from now on for a
1088    /// database in `directory` (an existing directory, matched by its
1089    /// canonical path), until the returned hold is released or dropped. A
1090    /// held registration has created its source and blocks in registering
1091    /// it; once released it registers, without ever notifying.
1092    pub fn hold_registrations_in(directory: &Path) -> std::io::Result<HeldRegistrations> {
1093        let directory = directory.canonicalize()?;
1094        let hold = Arc::new(Hold::default());
1095        let mut holds = lock(holds());
1096        if holds.contains_key(&directory) {
1097            return Err(std::io::Error::new(
1098                std::io::ErrorKind::AlreadyExists,
1099                format!("{} is already held", directory.display()),
1100            ));
1101        }
1102        holds.insert(directory.clone(), Arc::clone(&hold));
1103        Ok(HeldRegistrations { directory, hold })
1104    }
1105
1106    impl HeldRegistrations {
1107        /// Wait, at most `bound`, until `count` registrations have entered
1108        /// the hold. Whether they had.
1109        pub fn wait_entered(&self, count: usize, bound: Duration) -> bool {
1110            let deadline = Instant::now() + bound;
1111            let mut gate = lock(&self.hold.gate);
1112            while gate.entered < count {
1113                let Some(left) = deadline.checked_duration_since(Instant::now()) else {
1114                    return false;
1115                };
1116                gate = self
1117                    .hold
1118                    .changed
1119                    .wait_timeout(gate, left)
1120                    .unwrap_or_else(PoisonError::into_inner)
1121                    .0;
1122            }
1123            true
1124        }
1125
1126        /// Let every held registration register, and every later one in the
1127        /// directory register at once.
1128        pub fn release(&self) {
1129            lock(&self.hold.gate).released = true;
1130            self.hold.changed.notify_all();
1131        }
1132
1133        /// Sources created through this hold and not dropped yet.
1134        pub fn live_sources(&self) -> usize {
1135            self.hold.live_sources.load(Ordering::SeqCst)
1136        }
1137
1138        /// Sources created through this hold and dropped since.
1139        pub fn dropped_sources(&self) -> usize {
1140            self.hold.dropped_sources.load(Ordering::SeqCst)
1141        }
1142    }
1143
1144    impl Drop for HeldRegistrations {
1145        fn drop(&mut self) {
1146            lock(holds()).remove(&self.directory);
1147            self.release();
1148        }
1149    }
1150
1151    /// The held backend for a watch of `parent`, while a hold covers it.
1152    pub(super) fn held_backend(parent: &Path) -> Option<Box<dyn WatchBackend>> {
1153        let hold = lock(holds()).get(parent).cloned()?;
1154        Some(Box::new(HeldBackend(hold)))
1155    }
1156
1157    struct HeldBackend(Arc<Hold>);
1158
1159    impl WatchBackend for HeldBackend {
1160        fn create(
1161            &self,
1162            _targets: &WatchTargets,
1163            wake: mpsc::Sender<()>,
1164        ) -> Result<Box<dyn WatchSource>, String> {
1165            self.0.live_sources.fetch_add(1, Ordering::SeqCst);
1166            Ok(Box::new(HeldSource {
1167                hold: Arc::clone(&self.0),
1168                _wake: wake,
1169            }))
1170        }
1171    }
1172
1173    struct HeldSource {
1174        hold: Arc<Hold>,
1175        /// Kept for the source's life, as a real source keeps its wake.
1176        _wake: mpsc::Sender<()>,
1177    }
1178
1179    impl WatchSource for HeldSource {
1180        fn watch(&mut self, _directory: &Path) -> Result<(), String> {
1181            let mut gate = lock(&self.hold.gate);
1182            gate.entered += 1;
1183            self.hold.changed.notify_all();
1184            while !gate.released {
1185                gate = self
1186                    .hold
1187                    .changed
1188                    .wait(gate)
1189                    .unwrap_or_else(PoisonError::into_inner);
1190            }
1191            Ok(())
1192        }
1193    }
1194
1195    impl Drop for HeldSource {
1196        fn drop(&mut self) {
1197            self.hold.live_sources.fetch_sub(1, Ordering::SeqCst);
1198            self.hold.dropped_sources.fetch_add(1, Ordering::SeqCst);
1199        }
1200    }
1201
1202    /// Observed directories, by canonical path.
1203    static OBSERVATIONS: OnceLock<Mutex<HashMap<PathBuf, Arc<Observation>>>> = OnceLock::new();
1204
1205    fn observations() -> &'static Mutex<HashMap<PathBuf, Arc<Observation>>> {
1206        OBSERVATIONS.get_or_init(Mutex::default)
1207    }
1208
1209    #[derive(Default)]
1210    struct Observation {
1211        registered: Mutex<usize>,
1212        changed: Condvar,
1213        /// Report a database's WAL created or removed also as modified.
1214        wal_lifecycle_as_modified: AtomicBool,
1215        drops: Mutex<SourceDrops>,
1216        drops_changed: Condvar,
1217    }
1218
1219    /// The observed sources' drops, while a test holds them.
1220    #[derive(Default)]
1221    struct SourceDrops {
1222        held: bool,
1223        finished: usize,
1224    }
1225
1226    /// Observes the notification registrations of the watches on databases
1227    /// in one directory: they register with the platform as usual, and this
1228    /// counts the ones that did. Dropping it ends the observation.
1229    pub struct ObservedRegistrations {
1230        directory: PathBuf,
1231        observation: Arc<Observation>,
1232    }
1233
1234    /// Observe the registration of every watch started from now on for a
1235    /// database in `directory` (an existing directory, matched by its
1236    /// canonical path), so a test can order its own file activity after a
1237    /// watch's registration ([`ObservedRegistrations::wait_registered`]).
1238    pub fn observe_registrations_in(directory: &Path) -> std::io::Result<ObservedRegistrations> {
1239        let directory = directory.canonicalize()?;
1240        let observation = Arc::new(Observation::default());
1241        let mut observations = lock(observations());
1242        if observations.contains_key(&directory) {
1243            return Err(std::io::Error::new(
1244                std::io::ErrorKind::AlreadyExists,
1245                format!("{} is already observed", directory.display()),
1246            ));
1247        }
1248        observations.insert(directory.clone(), Arc::clone(&observation));
1249        Ok(ObservedRegistrations {
1250            directory,
1251            observation,
1252        })
1253    }
1254
1255    impl ObservedRegistrations {
1256        /// From now on, the observed watches also report a database's WAL
1257        /// created or removed as modified (`Modify(Any)`), as
1258        /// ReadDirectoryChangesW can report a new or removed file, so a
1259        /// connection's own open and close notify a WAL change. Each report
1260        /// still passes the watch's relevance filter.
1261        pub fn report_wal_lifecycle_as_modified(&self) {
1262            self.observation
1263                .wal_lifecycle_as_modified
1264                .store(true, Ordering::SeqCst);
1265        }
1266
1267        /// From now on, dropping an observed source blocks until
1268        /// [`Self::release_source_drops`], as a macOS FSEvents watcher's drop
1269        /// waits on the file-system event service.
1270        pub fn hold_source_drops(&self) {
1271            lock(&self.observation.drops).held = true;
1272        }
1273
1274        /// Let every held source drop, and every later one at once.
1275        pub fn release_source_drops(&self) {
1276            lock(&self.observation.drops).held = false;
1277            self.observation.drops_changed.notify_all();
1278        }
1279
1280        /// Observed sources dropped so far.
1281        pub fn source_drops_finished(&self) -> usize {
1282            lock(&self.observation.drops).finished
1283        }
1284
1285        /// Wait, at most `bound`, until `count` observed sources have been
1286        /// dropped. Whether they had.
1287        pub fn wait_source_drops_finished(&self, count: usize, bound: Duration) -> bool {
1288            let deadline = Instant::now() + bound;
1289            let mut drops = lock(&self.observation.drops);
1290            while drops.finished < count {
1291                let Some(left) = deadline.checked_duration_since(Instant::now()) else {
1292                    return false;
1293                };
1294                drops = self
1295                    .observation
1296                    .drops_changed
1297                    .wait_timeout(drops, left)
1298                    .unwrap_or_else(PoisonError::into_inner)
1299                    .0;
1300            }
1301            true
1302        }
1303
1304        /// Wait, at most `bound`, until `count` registrations have
1305        /// registered. Whether they had.
1306        pub fn wait_registered(&self, count: usize, bound: Duration) -> bool {
1307            let deadline = Instant::now() + bound;
1308            let mut registered = lock(&self.observation.registered);
1309            while *registered < count {
1310                let Some(left) = deadline.checked_duration_since(Instant::now()) else {
1311                    return false;
1312                };
1313                registered = self
1314                    .observation
1315                    .changed
1316                    .wait_timeout(registered, left)
1317                    .unwrap_or_else(PoisonError::into_inner)
1318                    .0;
1319            }
1320            true
1321        }
1322    }
1323
1324    impl Drop for ObservedRegistrations {
1325        fn drop(&mut self) {
1326            lock(observations()).remove(&self.directory);
1327            self.release_source_drops();
1328        }
1329    }
1330
1331    /// The platform backend, observed, for a watch of `parent` while an
1332    /// observation covers it.
1333    pub(super) fn observed_backend(parent: &Path) -> Option<Box<dyn WatchBackend>> {
1334        let observation = lock(observations()).get(parent).cloned()?;
1335        Some(Box::new(ObservedBackend(observation)))
1336    }
1337
1338    struct ObservedBackend(Arc<Observation>);
1339
1340    impl WatchBackend for ObservedBackend {
1341        fn create(
1342            &self,
1343            targets: &WatchTargets,
1344            wake: mpsc::Sender<()>,
1345        ) -> Result<Box<dyn WatchSource>, String> {
1346            let inner = if self.0.wal_lifecycle_as_modified.load(Ordering::SeqCst) {
1347                wal_lifecycle_as_modified_source(targets, wake)?
1348            } else {
1349                super::NotifyBackend.create(targets, wake)?
1350            };
1351            Ok(Box::new(ObservedSource {
1352                inner: Some(inner),
1353                observation: Arc::clone(&self.0),
1354            }))
1355        }
1356    }
1357
1358    /// The platform watcher, with a database's WAL created or removed also
1359    /// reported as modified; every report passes the relevance filter.
1360    fn wal_lifecycle_as_modified_source(
1361        targets: &WatchTargets,
1362        wake: mpsc::Sender<()>,
1363    ) -> Result<Box<dyn WatchSource>, String> {
1364        use notify::EventKind;
1365        let parent = targets.parent.clone();
1366        let watched_paths = targets.watched_paths.clone();
1367        let wals: Vec<PathBuf> = watched_paths
1368            .iter()
1369            .map(|path| super::sidecar_path(path, "-wal"))
1370            .filter(|wal| watched_paths.contains(wal))
1371            .collect();
1372        let watcher = notify::recommended_watcher(move |result: notify::Result<notify::Event>| {
1373            let Ok(event) = result else {
1374                log_raw_event(&parent, &result, false);
1375                return;
1376            };
1377            let mut reports = vec![event.clone()];
1378            if matches!(event.kind, EventKind::Create(_) | EventKind::Remove(_)) {
1379                for wal in event.paths.iter().filter(|path| wals.contains(path)) {
1380                    reports.push(
1381                        notify::Event::new(EventKind::Modify(notify::event::ModifyKind::Any))
1382                            .add_path(wal.clone()),
1383                    );
1384                }
1385            }
1386            let relevant = reports
1387                .iter()
1388                .any(|report| super::sqlite_watch_event_relevant(report, &parent, &watched_paths));
1389            log_raw_event(&parent, &Ok(event), relevant);
1390            if relevant {
1391                let _ = wake.send(());
1392            }
1393        })
1394        .map_err(|error| format!("cannot create the sqlite change watcher: {error}"))?;
1395        Ok(Box::new(super::NotifySource(watcher)))
1396    }
1397
1398    struct ObservedSource {
1399        /// `None` once dropped.
1400        inner: Option<Box<dyn WatchSource>>,
1401        observation: Arc<Observation>,
1402    }
1403
1404    impl WatchSource for ObservedSource {
1405        fn watch(&mut self, directory: &Path) -> Result<(), String> {
1406            if let Some(inner) = self.inner.as_mut() {
1407                inner.watch(directory)?;
1408            }
1409            *lock(&self.observation.registered) += 1;
1410            self.observation.changed.notify_all();
1411            Ok(())
1412        }
1413
1414        /// The platform's answer, or that a test holds this source's drop.
1415        fn teardown_waits(&self) -> bool {
1416            self.inner
1417                .as_ref()
1418                .is_some_and(|inner| inner.teardown_waits())
1419                || lock(&self.observation.drops).held
1420        }
1421    }
1422
1423    impl Drop for ObservedSource {
1424        fn drop(&mut self) {
1425            let mut drops = lock(&self.observation.drops);
1426            while drops.held {
1427                drops = self
1428                    .observation
1429                    .drops_changed
1430                    .wait(drops)
1431                    .unwrap_or_else(PoisonError::into_inner);
1432            }
1433            drop(drops);
1434            drop(self.inner.take());
1435            lock(&self.observation.drops).finished += 1;
1436            self.observation.drops_changed.notify_all();
1437        }
1438    }
1439}
1440
1441#[cfg(test)]
1442#[allow(clippy::expect_used, clippy::unwrap_used)]
1443mod tests {
1444    use super::*;
1445    use std::sync::atomic::AtomicUsize;
1446
1447    /// Wait, within the registration bound, until the watch's registration
1448    /// ends.
1449    fn settled_registration(watch: &SqliteChangeWatch) -> SqliteWatchRegistration {
1450        let deadline = Instant::now() + REGISTRATION_BOUND;
1451        loop {
1452            let registration = watch.registration();
1453            if registration != SqliteWatchRegistration::Pending || Instant::now() >= deadline {
1454                return registration;
1455            }
1456            thread::sleep(Duration::from_millis(2));
1457        }
1458    }
1459
1460    fn retained_in(directory: &Path) -> Vec<SqliteWatchRetainedRegistration> {
1461        retained_registrations()
1462            .into_iter()
1463            .filter(|retained| retained.directory == directory)
1464            .collect()
1465    }
1466
1467    /// Wait, within the hang guard, for a Changed tick, skipping the sweeps: a
1468    /// sweep that ticked while the registration was pending (a macOS FSEvents
1469    /// registration can outlast the sweep) comes first. A source that never
1470    /// notifies leaves sweeps only, and fails.
1471    fn next_changed(tick_rx: &mpsc::Receiver<SqliteWatchTick>) -> Result<(), String> {
1472        let deadline = Instant::now() + HANG_GUARD;
1473        let mut sweeps = 0_usize;
1474        loop {
1475            match tick_rx.recv_timeout(deadline.saturating_duration_since(Instant::now())) {
1476                Ok(SqliteWatchTick::Changed) => return Ok(()),
1477                Ok(SqliteWatchTick::Sweep) => sweeps += 1,
1478                Err(error) => {
1479                    return Err(format!(
1480                        "no Changed tick within the hang guard, {sweeps} sweeps: {error}"
1481                    ));
1482                }
1483            }
1484        }
1485    }
1486
1487    /// Whether a Changed tick comes within `window`, skipping sweeps.
1488    fn changed_within(tick_rx: &mpsc::Receiver<SqliteWatchTick>, window: Duration) -> bool {
1489        let deadline = Instant::now() + window;
1490        loop {
1491            match tick_rx.recv_timeout(deadline.saturating_duration_since(Instant::now())) {
1492                Ok(SqliteWatchTick::Changed) => return true,
1493                Ok(SqliteWatchTick::Sweep) => {}
1494                Err(_) => return false,
1495            }
1496        }
1497    }
1498
1499    /// Append `bytes` to the existing file at `path` and close it, as a
1500    /// commit appends frames to a WAL that a connection holds open.
1501    fn append(path: &Path, bytes: &[u8]) {
1502        use std::io::Write as _;
1503        let mut file = std::fs::OpenOptions::new().append(true).open(path).unwrap();
1504        file.write_all(bytes).unwrap();
1505        drop(file);
1506        log_row(
1507            path,
1508            format_args!("appended {} bytes and closed", bytes.len()),
1509        );
1510    }
1511
1512    /// Log a row's step for the database file or sidecar `path`, naming its
1513    /// canonical directory as the notifications for it are logged (see
1514    /// [`test_support::RAW_EVENT_LOG`]).
1515    fn log_row(path: &Path, step: std::fmt::Arguments<'_>) {
1516        let dir = path
1517            .parent()
1518            .and_then(|parent| parent.canonicalize().ok())
1519            .unwrap_or_default();
1520        test_support::log_raw(format_args!(
1521            "row dir={} file={:?}: {step}",
1522            dir.display(),
1523            path.file_name().unwrap_or_default()
1524        ));
1525    }
1526
1527    /// A WAL database's WAL exists before the watch starts, as it does while
1528    /// any connection holds the database open, so a write to it is a data
1529    /// change only. (Its creation is a connection's own open, which ticks
1530    /// nothing: see [`sqlite_watch_event_relevant`].)
1531    fn assert_database_write_ticks_changed(db: &Path) {
1532        log_row(db, format_args!("a write row starts on {}", db.display()));
1533        std::fs::write(db, b"").unwrap();
1534        let wal = sidecar_path(db, "-wal");
1535        std::fs::write(&wal, b"").unwrap();
1536        let (tick_tx, tick_rx) = mpsc::channel();
1537        let watch = start_sqlite_change_watch(db, "test-watch", SQLITE_WATCH_SWEEP, move |tick| {
1538            let _ = tick_tx.send(tick);
1539            SqliteWatchControl::Handled
1540        })
1541        .expect("watch starts");
1542        // Notifications are registered after the start returns: a write
1543        // before that is seen by the sweep alone.
1544        assert_eq!(
1545            settled_registration(&watch),
1546            SqliteWatchRegistration::Registered
1547        );
1548        append(&wal, b"frame");
1549        next_changed(&tick_rx).expect("the write is observed within the hang guard");
1550    }
1551
1552    #[test]
1553    fn a_write_to_the_database_ticks_changed() {
1554        let dir = tempfile::tempdir().unwrap();
1555        assert_database_write_ticks_changed(&dir.path().join("store.sqlite3"));
1556    }
1557
1558    #[cfg(unix)]
1559    #[test]
1560    fn symlinked_directory_targets_match_canonical_events_for_missing_files() {
1561        let dir = tempfile::tempdir().unwrap();
1562        let actual = dir.path().join("actual");
1563        let alias = dir.path().join("alias");
1564        std::fs::create_dir(&actual).unwrap();
1565        std::os::unix::fs::symlink(&actual, &alias).unwrap();
1566        let db = alias.join("store.sqlite3");
1567        let (parent, watched_paths) = sqlite_watch_targets(&db).unwrap();
1568        let canonical_parent = actual.canonicalize().unwrap();
1569        assert_eq!(parent, canonical_parent);
1570        assert_eq!(
1571            watched_paths,
1572            sqlite_watch_paths(&canonical_parent.join("store.sqlite3"))
1573        );
1574        assert!(!db.exists());
1575        for path in &watched_paths {
1576            assert!(!path.exists());
1577        }
1578        let [database, wal, _index] = <&[PathBuf; 3]>::try_from(watched_paths.as_slice())
1579            .expect("the database and its two sidecars");
1580        for kind in [
1581            notify::EventKind::Create(notify::event::CreateKind::File),
1582            notify::EventKind::Remove(notify::event::RemoveKind::File),
1583        ] {
1584            let event = notify::Event::new(kind).add_path(database.clone());
1585            assert!(sqlite_watch_event_relevant(&event, &parent, &watched_paths));
1586        }
1587        let append = notify::Event::new(notify::EventKind::Modify(
1588            notify::event::ModifyKind::Data(notify::event::DataChange::Any),
1589        ))
1590        .add_path(wal.clone());
1591        assert!(sqlite_watch_event_relevant(
1592            &append,
1593            &parent,
1594            &watched_paths
1595        ));
1596        for path in [
1597            canonical_parent.join("other.sqlite3-wal"),
1598            canonical_parent.join("other").join("store.sqlite3-wal"),
1599        ] {
1600            let event = notify::Event::new(notify::EventKind::Any).add_path(path);
1601            assert!(!sqlite_watch_event_relevant(
1602                &event,
1603                &parent,
1604                &watched_paths
1605            ));
1606        }
1607        let access = notify::Event::new(notify::EventKind::Access(notify::event::AccessKind::Any))
1608            .add_path(watched_paths[0].clone());
1609        assert!(!sqlite_watch_event_relevant(
1610            &access,
1611            &parent,
1612            &watched_paths
1613        ));
1614    }
1615
1616    /// The `notify` kinds one platform backend reports for a WAL database's
1617    /// files: a connection's own open and close, and a commit.
1618    struct BackendShapes {
1619        /// The first connection to open the database creates its WAL and the
1620        /// WAL index.
1621        created: &'static [notify::EventKind],
1622        /// It writes the index (its truncation and size) and can touch the
1623        /// sidecars' metadata.
1624        written: &'static [notify::EventKind],
1625        /// The last connection to close removes both.
1626        removed: &'static [notify::EventKind],
1627        /// A commit appends to the WAL; without a WAL it writes the database.
1628        appended: &'static [notify::EventKind],
1629    }
1630
1631    /// A connection's own open and close of a WAL database are no change,
1632    /// whichever kinds the backend reports them as; a commit is, and so is
1633    /// any event on the database file or the directory itself.
1634    fn assert_only_a_commit_is_a_change(shapes: &BackendShapes) {
1635        let parent = PathBuf::from("/data");
1636        let database = parent.join("store.sqlite3");
1637        let watched_paths = sqlite_watch_paths(&database);
1638        let wal = sidecar_path(&database, "-wal");
1639        let index = sidecar_path(&database, "-shm");
1640        let relevant = |kind: notify::EventKind, path: &Path| {
1641            sqlite_watch_event_relevant(
1642                &notify::Event::new(kind).add_path(path.to_path_buf()),
1643                &parent,
1644                &watched_paths,
1645            )
1646        };
1647        for &kind in shapes.created.iter().chain(shapes.removed) {
1648            assert!(
1649                !relevant(kind, &wal),
1650                "the WAL created or removed is no change: {kind:?}"
1651            );
1652        }
1653        for &kind in shapes
1654            .created
1655            .iter()
1656            .chain(shapes.written)
1657            .chain(shapes.removed)
1658        {
1659            assert!(
1660                !relevant(kind, &index),
1661                "nothing on the WAL index is a change: {kind:?}"
1662            );
1663        }
1664        for &kind in shapes.appended {
1665            assert!(relevant(kind, &wal), "a WAL append is a change: {kind:?}");
1666            assert!(
1667                relevant(kind, &database),
1668                "a database write is a change: {kind:?}"
1669            );
1670        }
1671        for &kind in shapes.created.iter().chain(shapes.removed) {
1672            assert!(
1673                relevant(kind, &database),
1674                "the database created or removed is a change: {kind:?}"
1675            );
1676            assert!(
1677                relevant(kind, &parent),
1678                "the directory itself is a change: {kind:?}"
1679            );
1680        }
1681        let rescan = notify::Event::new(notify::EventKind::Other);
1682        assert!(
1683            sqlite_watch_event_relevant(&rescan, &parent, &watched_paths),
1684            "a rescan names no path and is a change"
1685        );
1686    }
1687
1688    /// inotify (Linux): IN_CREATE, IN_MODIFY, IN_ATTRIB and IN_DELETE.
1689    #[test]
1690    fn inotify_kinds_of_a_connections_own_open_and_close_are_no_change() {
1691        use notify::EventKind::{Create, Modify, Remove};
1692        use notify::event::{CreateKind, DataChange, MetadataKind, ModifyKind, RemoveKind};
1693        assert_only_a_commit_is_a_change(&BackendShapes {
1694            created: &[Create(CreateKind::File)],
1695            written: &[
1696                Modify(ModifyKind::Data(DataChange::Any)),
1697                Modify(ModifyKind::Metadata(MetadataKind::Any)),
1698            ],
1699            removed: &[Remove(RemoveKind::File)],
1700            appended: &[Modify(ModifyKind::Data(DataChange::Any))],
1701        });
1702    }
1703
1704    /// FSEvents (macOS): one event per flag of a record, so a record that
1705    /// merges the WAL's creation with an append still carries the append's
1706    /// data change.
1707    #[test]
1708    fn fsevents_kinds_of_a_connections_own_open_and_close_are_no_change() {
1709        use notify::EventKind::{Create, Modify, Remove};
1710        use notify::event::{CreateKind, DataChange, MetadataKind, ModifyKind, RemoveKind};
1711        assert_only_a_commit_is_a_change(&BackendShapes {
1712            created: &[Create(CreateKind::File), Create(CreateKind::Any)],
1713            written: &[
1714                Modify(ModifyKind::Data(DataChange::Content)),
1715                Modify(ModifyKind::Metadata(MetadataKind::Any)),
1716                Modify(ModifyKind::Metadata(MetadataKind::Ownership)),
1717                Modify(ModifyKind::Metadata(MetadataKind::Extended)),
1718            ],
1719            removed: &[Remove(RemoveKind::File), Remove(RemoveKind::Any)],
1720            appended: &[Modify(ModifyKind::Data(DataChange::Content))],
1721        });
1722        let parent = PathBuf::from("/data");
1723        let watched_paths = sqlite_watch_paths(&parent.join("store.sqlite3"));
1724        let merged = [
1725            Create(CreateKind::File),
1726            Modify(ModifyKind::Metadata(MetadataKind::Any)),
1727            Modify(ModifyKind::Data(DataChange::Content)),
1728        ];
1729        assert!(
1730            merged.into_iter().any(|kind| sqlite_watch_event_relevant(
1731                &notify::Event::new(kind).add_path(watched_paths[1].clone()),
1732                &parent,
1733                &watched_paths,
1734            )),
1735            "a merged record of the WAL's creation and an append is a change"
1736        );
1737    }
1738
1739    /// ReadDirectoryChangesW (Windows): added, modified and removed, with no
1740    /// finer kinds, so every modification of the WAL stays a change.
1741    #[test]
1742    fn read_directory_changes_kinds_of_a_connections_own_open_and_close_are_no_change() {
1743        use notify::EventKind::{Any, Create, Modify, Remove};
1744        use notify::event::{CreateKind, ModifyKind, RemoveKind};
1745        assert_only_a_commit_is_a_change(&BackendShapes {
1746            created: &[Create(CreateKind::Any)],
1747            written: &[Modify(ModifyKind::Any)],
1748            removed: &[Remove(RemoveKind::Any)],
1749            appended: &[Modify(ModifyKind::Any), Any],
1750        });
1751    }
1752
1753    /// On a WAL database, a connection's own open (the first: it creates the
1754    /// WAL and its index) and close (the last: it removes them) tick no
1755    /// watch; its commit does. inotify reports all of it.
1756    #[cfg(target_os = "linux")]
1757    #[test]
1758    fn a_connections_own_open_and_close_tick_nothing_but_its_commit_does() {
1759        let dir = tempfile::tempdir().unwrap();
1760        let db = wal_database(dir.path());
1761        let wal = sidecar_path(&db, "-wal");
1762        let index = sidecar_path(&db, "-shm");
1763        assert!(
1764            !wal.exists() && !index.exists(),
1765            "the last close removed both"
1766        );
1767        let (tick_tx, tick_rx) = mpsc::channel();
1768        let watch =
1769            start_sqlite_change_watch(&db, "test-own-open", Duration::from_hours(1), move |tick| {
1770                let _ = tick_tx.send(tick);
1771                SqliteWatchControl::Handled
1772            })
1773            .expect("watch starts");
1774        assert_eq!(
1775            settled_registration(&watch),
1776            SqliteWatchRegistration::Registered
1777        );
1778        {
1779            let conn = crate::open(&db, crate::ConnectionProfile::PRIMARY).unwrap();
1780            let rows: i64 = conn
1781                .query_row("SELECT COUNT(*) FROM t", [], |row| row.get(0))
1782                .unwrap();
1783            assert_eq!(rows, 1);
1784            assert!(
1785                wal.exists() && index.exists(),
1786                "the first connection created the WAL and its index"
1787            );
1788        }
1789        assert!(
1790            !wal.exists() && !index.exists(),
1791            "the last connection removed them"
1792        );
1793        // The watch coalesces for 10 ms: a tick of the open or the close
1794        // would have come long before this.
1795        assert_eq!(
1796            tick_rx.recv_timeout(Duration::from_millis(500)),
1797            Err(mpsc::RecvTimeoutError::Timeout),
1798            "a connection's own open and close are no change"
1799        );
1800        {
1801            let conn = crate::open(&db, crate::ConnectionProfile::PRIMARY).unwrap();
1802            conn.execute("INSERT INTO t (v) VALUES (2)", []).unwrap();
1803        }
1804        assert_eq!(
1805            tick_rx
1806                .recv_timeout(HANG_GUARD)
1807                .expect("the commit ticks within the hang guard"),
1808            SqliteWatchTick::Changed
1809        );
1810    }
1811
1812    /// How long a commit row waits for a commit's tick before its connection
1813    /// closes. What it saw is logged (see [`test_support::RAW_EVENT_LOG`]),
1814    /// never required: a platform can report a write only when the file
1815    /// written closes.
1816    const BEFORE_CLOSE_WINDOW: Duration = Duration::from_secs(2);
1817
1818    /// A commit while another connection (a kept reader) holds a WAL database
1819    /// open appends to the existing WAL and ticks the watch. The committing
1820    /// connection closes after its commit, as the stores' connections do.
1821    #[test]
1822    fn a_commit_to_an_existing_wal_ticks_changed() {
1823        let dir = tempfile::tempdir().unwrap();
1824        let db = wal_database(dir.path());
1825        log_row(
1826            &db,
1827            format_args!("a_commit_to_an_existing_wal_ticks_changed starts"),
1828        );
1829        let wal = sidecar_path(&db, "-wal");
1830        let kept = crate::open(&db, crate::ConnectionProfile::PRIMARY).unwrap();
1831        let rows: i64 = kept
1832            .query_row("SELECT COUNT(*) FROM t", [], |row| row.get(0))
1833            .unwrap();
1834        assert_eq!(rows, 1);
1835        assert!(wal.exists(), "the kept connection holds the WAL open");
1836        let (tick_tx, tick_rx) = mpsc::channel();
1837        let watch = start_sqlite_change_watch(
1838            &db,
1839            "test-commit-existing-wal",
1840            Duration::from_hours(1),
1841            move |tick| {
1842                let _ = tick_tx.send(tick);
1843                SqliteWatchControl::Handled
1844            },
1845        )
1846        .expect("watch starts");
1847        assert_eq!(
1848            settled_registration(&watch),
1849            SqliteWatchRegistration::Registered
1850        );
1851        let writer = crate::open(&db, crate::ConnectionProfile::PRIMARY).unwrap();
1852        writer.execute("INSERT INTO t (v) VALUES (2)", []).unwrap();
1853        let wal_len = std::fs::metadata(&wal).map(|metadata| metadata.len());
1854        log_row(&wal, format_args!("committed, WAL length {wal_len:?}"));
1855        let before_close = changed_within(&tick_rx, BEFORE_CLOSE_WINDOW);
1856        log_row(
1857            &wal,
1858            format_args!("Changed before the writer closed: {before_close}"),
1859        );
1860        drop(writer);
1861        if !before_close {
1862            next_changed(&tick_rx).expect("the commit ticks once its connection closed");
1863            log_row(&wal, format_args!("Changed after the writer closed"));
1864        }
1865        drop(kept);
1866    }
1867
1868    /// A connection that opens a WAL database no other connection holds open
1869    /// creates the WAL, and its commit at once appends to that new WAL: the
1870    /// watch ticks. Whether it ticks before the connection closes (its close,
1871    /// the last, then checkpoints into the database file and removes the
1872    /// WAL) is logged, with every notification (see
1873    /// [`test_support::RAW_EVENT_LOG`]).
1874    #[test]
1875    fn a_commit_that_creates_the_wal_ticks_changed() {
1876        let dir = tempfile::tempdir().unwrap();
1877        let db = wal_database(dir.path());
1878        log_row(
1879            &db,
1880            format_args!("a_commit_that_creates_the_wal_ticks_changed starts"),
1881        );
1882        let wal = sidecar_path(&db, "-wal");
1883        assert!(!wal.exists(), "no connection holds the database open");
1884        let (tick_tx, tick_rx) = mpsc::channel();
1885        let watch = start_sqlite_change_watch(
1886            &db,
1887            "test-commit-creates-wal",
1888            Duration::from_hours(1),
1889            move |tick| {
1890                let _ = tick_tx.send(tick);
1891                SqliteWatchControl::Handled
1892            },
1893        )
1894        .expect("watch starts");
1895        assert_eq!(
1896            settled_registration(&watch),
1897            SqliteWatchRegistration::Registered
1898        );
1899        let writer = crate::open(&db, crate::ConnectionProfile::PRIMARY).unwrap();
1900        writer.execute("INSERT INTO t (v) VALUES (2)", []).unwrap();
1901        let wal_len = std::fs::metadata(&wal).map(|metadata| metadata.len());
1902        log_row(&wal, format_args!("committed, WAL length {wal_len:?}"));
1903        let before_close = changed_within(&tick_rx, BEFORE_CLOSE_WINDOW);
1904        log_row(
1905            &wal,
1906            format_args!("Changed before the writer closed: {before_close}"),
1907        );
1908        drop(writer);
1909        assert!(!wal.exists(), "the last connection removed the WAL");
1910        if !before_close {
1911            next_changed(&tick_rx).expect("the commit ticks once its connection closed");
1912            log_row(&wal, format_args!("Changed after the writer closed"));
1913        }
1914    }
1915
1916    #[cfg(unix)]
1917    #[test]
1918    fn a_write_through_a_symlinked_directory_ticks_changed() {
1919        let dir = tempfile::tempdir().unwrap();
1920        let actual = dir.path().join("actual");
1921        let alias = dir.path().join("alias");
1922        std::fs::create_dir(&actual).unwrap();
1923        std::os::unix::fs::symlink(&actual, &alias).unwrap();
1924        assert_database_write_ticks_changed(&alias.join("store.sqlite3"));
1925    }
1926
1927    /// The sweep ticks with no notification. Once the watch registers, a
1928    /// platform can report the watched directory's own recent creation (macOS
1929    /// FSEvents does, and an event on the directory itself is relevant), so
1930    /// Changed ticks before the first sweep are skipped.
1931    #[test]
1932    fn the_sweep_ticks_without_a_notification() {
1933        let dir = tempfile::tempdir().unwrap();
1934        let db = dir.path().join("store.sqlite3");
1935        let (tick_tx, tick_rx) = mpsc::channel();
1936        let _watch =
1937            start_sqlite_change_watch(&db, "test-sweep", Duration::from_millis(50), move |tick| {
1938                let _ = tick_tx.send(tick);
1939                SqliteWatchControl::Handled
1940            })
1941            .expect("watch starts");
1942        next_sweep(&tick_rx).expect("the sweep ticks within the hang guard");
1943    }
1944
1945    /// Wait, within the hang guard, for a Sweep tick, skipping Changed ticks.
1946    fn next_sweep(tick_rx: &mpsc::Receiver<SqliteWatchTick>) -> Result<(), String> {
1947        let deadline = Instant::now() + HANG_GUARD;
1948        let mut changed = 0_usize;
1949        loop {
1950            match tick_rx.recv_timeout(deadline.saturating_duration_since(Instant::now())) {
1951                Ok(SqliteWatchTick::Sweep) => return Ok(()),
1952                Ok(SqliteWatchTick::Changed) => changed += 1,
1953                Err(error) => {
1954                    return Err(format!(
1955                        "no Sweep tick within the hang guard, {changed} Changed: {error}"
1956                    ));
1957                }
1958            }
1959        }
1960    }
1961
1962    #[test]
1963    fn stop_ends_the_worker() {
1964        let dir = tempfile::tempdir().unwrap();
1965        let db = dir.path().join("store.sqlite3");
1966        let (tick_tx, tick_rx) = mpsc::channel();
1967        let _watch =
1968            start_sqlite_change_watch(&db, "test-stop", Duration::from_millis(20), move |tick| {
1969                let _ = tick_tx.send(tick);
1970                SqliteWatchControl::Stop
1971            })
1972            .expect("watch starts");
1973        tick_rx
1974            .recv_timeout(Duration::from_secs(10))
1975            .expect("one tick");
1976        assert!(
1977            tick_rx.recv_timeout(Duration::from_millis(500)).is_err(),
1978            "no tick after Stop: the worker ended (its sender is gone)"
1979        );
1980    }
1981
1982    /// A WAL database holding one row, as the stores keep theirs.
1983    fn wal_database(dir: &Path) -> PathBuf {
1984        let db = dir.join("store.sqlite3");
1985        let conn = crate::open(&db, crate::ConnectionProfile::PRIMARY).unwrap();
1986        conn.execute_batch("CREATE TABLE t (v INTEGER); INSERT INTO t (v) VALUES (1);")
1987            .unwrap();
1988        db
1989    }
1990
1991    /// Counts ticks, and reports once when the worker drops its tick closure,
1992    /// which it does as it returns.
1993    struct TickProbe {
1994        ticks: Arc<AtomicUsize>,
1995        returned: mpsc::Sender<()>,
1996    }
1997
1998    impl TickProbe {
1999        fn tick(&self) {
2000            self.ticks.fetch_add(1, Ordering::SeqCst);
2001        }
2002    }
2003
2004    impl Drop for TickProbe {
2005        fn drop(&mut self) {
2006            let _ = self.returned.send(());
2007        }
2008    }
2009
2010    #[test]
2011    fn stop_and_join_waits_for_the_tick_in_flight() {
2012        let dir = tempfile::tempdir().unwrap();
2013        let db = wal_database(dir.path());
2014        let (entered_tx, entered_rx) = mpsc::channel();
2015        let (release_tx, release_rx) = mpsc::channel::<()>();
2016        let finished = Arc::new(AtomicBool::new(false));
2017        let tick_finished = Arc::clone(&finished);
2018        let tick_db = db.clone();
2019        let mut watch = start_sqlite_change_watch(
2020            &db,
2021            "test-held-tick",
2022            Duration::from_millis(20),
2023            move |_tick| {
2024                if tick_finished.load(Ordering::SeqCst) {
2025                    return SqliteWatchControl::Handled;
2026                }
2027                // The held tick keeps a read connection open, as a store
2028                // catch-up does.
2029                let conn = crate::open(&tick_db, crate::ConnectionProfile::ReadOnly).unwrap();
2030                let _ = entered_tx.send(());
2031                let _ = release_rx.recv();
2032                let value: i64 = conn
2033                    .query_row("SELECT v FROM t", [], |row| row.get(0))
2034                    .unwrap();
2035                assert_eq!(value, 1);
2036                drop(conn);
2037                tick_finished.store(true, Ordering::SeqCst);
2038                SqliteWatchControl::Handled
2039            },
2040        )
2041        .expect("watch starts");
2042        entered_rx
2043            .recv_timeout(Duration::from_secs(10))
2044            .expect("a tick is held within the hang guard");
2045
2046        let (stopping_tx, stopping_rx) = mpsc::channel();
2047        let stopper_finished = Arc::clone(&finished);
2048        let stopper = thread::spawn(move || {
2049            let _ = stopping_tx.send(());
2050            let outcome = watch.stop_and_join();
2051            (outcome, stopper_finished.load(Ordering::SeqCst))
2052        });
2053        stopping_rx.recv().unwrap();
2054        release_tx.send(()).unwrap();
2055        let (outcome, finished_at_return) = stopper.join().unwrap();
2056
2057        assert!(
2058            finished_at_return,
2059            "stop_and_join returned while a tick still held its connection"
2060        );
2061        assert_eq!(outcome, SqliteWatchJoin::Joined);
2062        // The joined worker can never open the database again, so nothing
2063        // recreates a file while the directory goes: no sleep, no retry.
2064        std::fs::remove_dir_all(dir.path()).unwrap();
2065        assert!(!dir.path().exists());
2066    }
2067
2068    #[test]
2069    fn stop_and_join_on_the_worker_thread_is_unjoined_and_keeps_custody() {
2070        let dir = tempfile::tempdir().unwrap();
2071        let db = dir.path().join("store.sqlite3");
2072        let custody: Arc<std::sync::Mutex<Option<SqliteChangeWatch>>> = Arc::default();
2073        let tick_custody = Arc::clone(&custody);
2074        let (outcome_tx, outcome_rx) = mpsc::channel();
2075        let watch = start_sqlite_change_watch(
2076            &db,
2077            "test-self-stop",
2078            Duration::from_millis(20),
2079            move |_| {
2080                let outcome = tick_custody
2081                    .lock()
2082                    .unwrap()
2083                    .as_mut()
2084                    .map(SqliteChangeWatch::stop_and_join);
2085                if let Some(outcome) = outcome {
2086                    let _ = outcome_tx.send(outcome);
2087                }
2088                SqliteWatchControl::Handled
2089            },
2090        )
2091        .expect("watch starts");
2092        *custody.lock().unwrap() = Some(watch);
2093
2094        let on_worker = outcome_rx
2095            .recv_timeout(Duration::from_secs(10))
2096            .expect("a tick stops its own watch within the hang guard");
2097        assert_eq!(on_worker, SqliteWatchJoin::UnjoinedOnWorkerThread);
2098
2099        // The self-stop kept the join handle, so from here the worker is
2100        // really joined rather than reported as gone.
2101        let mut watch = custody
2102            .lock()
2103            .unwrap()
2104            .take()
2105            .expect("the watch stays in custody");
2106        assert!(!watch.on_worker_thread());
2107        assert_eq!(watch.stop_and_join(), SqliteWatchJoin::Joined);
2108        assert!(watch.is_finished());
2109        assert_eq!(
2110            watch.stop_and_join(),
2111            SqliteWatchJoin::Joined,
2112            "a repeated stop reports the recorded join"
2113        );
2114    }
2115
2116    #[test]
2117    fn dropping_the_watch_requests_the_stop_without_blocking() {
2118        let dir = tempfile::tempdir().unwrap();
2119        let db = dir.path().join("store.sqlite3");
2120        let (entered_tx, entered_rx) = mpsc::channel();
2121        let (release_tx, release_rx) = mpsc::channel::<()>();
2122        let (returned_tx, returned_rx) = mpsc::channel();
2123        let ticks = Arc::new(AtomicUsize::new(0));
2124        let probe = TickProbe {
2125            ticks: Arc::clone(&ticks),
2126            returned: returned_tx,
2127        };
2128        let watch =
2129            start_sqlite_change_watch(&db, "test-drop", Duration::from_millis(20), move |_| {
2130                probe.tick();
2131                let _ = entered_tx.send(());
2132                let _ = release_rx.recv();
2133                SqliteWatchControl::Handled
2134            })
2135            .expect("watch starts");
2136        entered_rx
2137            .recv_timeout(Duration::from_secs(10))
2138            .expect("a tick is held within the hang guard");
2139
2140        // Dropping never waits for the held tick: this thread releases the
2141        // tick only after the drop returned.
2142        drop(watch);
2143        release_tx.send(()).unwrap();
2144        returned_rx
2145            .recv_timeout(Duration::from_secs(10))
2146            .expect("the worker returns after the held tick");
2147        assert_eq!(
2148            ticks.load(Ordering::SeqCst),
2149            1,
2150            "no tick starts after the stop request"
2151        );
2152    }
2153
2154    /// How long a held registration may keep a start from returning, or the
2155    /// sweep from ticking, before the test calls it held.
2156    const HELD_BOUND: Duration = Duration::from_secs(5);
2157    const HANG_GUARD: Duration = Duration::from_secs(10);
2158    /// How long a row waits for a watch to register with the platform: a hang
2159    /// guard, not a latency bound. A macOS FSEvents registration can take tens
2160    /// of seconds ([`WatchSource::watch`]), so a minute; a registration that
2161    /// never returns still fails the row.
2162    const REGISTRATION_BOUND: Duration = Duration::from_mins(1);
2163    /// How long a row waits for released sources to finish their teardown:
2164    /// a hang guard, not a latency bound. Dropping a macOS FSEvents source
2165    /// joins its run-loop thread, which makes two round trips to the file
2166    /// system event service (the current event id, then a purge up to it)
2167    /// before it lets go ([`WatchSource::teardown_waits`]), and under load two
2168    /// of those teardowns in turn outlasted the 10 s guard. So a minute; a
2169    /// teardown that never ends still fails the row.
2170    const TEARDOWN_BOUND: Duration = Duration::from_mins(1);
2171
2172    #[test]
2173    fn a_held_registration_does_not_hold_the_start() {
2174        let dir = tempfile::tempdir().unwrap();
2175        let held = test_support::hold_registrations_in(dir.path()).unwrap();
2176        let db = dir.path().join("store.sqlite3");
2177        let (started_tx, started_rx) = mpsc::channel();
2178        let starter = thread::spawn(move || {
2179            let watch =
2180                start_sqlite_change_watch(&db, "test-held-start", SQLITE_WATCH_SWEEP, |_| {
2181                    SqliteWatchControl::Handled
2182                });
2183            let _ = started_tx.send(());
2184            watch
2185        });
2186        assert!(
2187            held.wait_entered(1, HANG_GUARD),
2188            "the registration reached its platform call"
2189        );
2190        let started = started_rx.recv_timeout(HELD_BOUND);
2191        held.release();
2192        let watch = starter.join().unwrap().expect("watch starts");
2193        assert_eq!(
2194            started,
2195            Ok(()),
2196            "the start waited for the held notification registration"
2197        );
2198        drop(watch);
2199    }
2200
2201    #[test]
2202    fn the_sweep_ticks_while_the_registration_is_held() {
2203        let dir = tempfile::tempdir().unwrap();
2204        let held = test_support::hold_registrations_in(dir.path()).unwrap();
2205        let db = dir.path().join("store.sqlite3");
2206        let (tick_tx, tick_rx) = mpsc::channel();
2207        let starter = thread::spawn(move || {
2208            start_sqlite_change_watch(
2209                &db,
2210                "test-held-sweep",
2211                Duration::from_millis(20),
2212                move |tick| {
2213                    let _ = tick_tx.send(tick);
2214                    SqliteWatchControl::Handled
2215                },
2216            )
2217        });
2218        assert!(
2219            held.wait_entered(1, HANG_GUARD),
2220            "the registration reached its platform call"
2221        );
2222        let tick = tick_rx.recv_timeout(HELD_BOUND);
2223        held.release();
2224        let watch = starter.join().unwrap().expect("watch starts");
2225        assert_eq!(
2226            tick,
2227            Ok(SqliteWatchTick::Sweep),
2228            "no sweep ran while the notification registration was held"
2229        );
2230        drop(watch);
2231    }
2232
2233    /// A source whose registration fails, as inotify's does past the user's
2234    /// watch limit.
2235    struct FailingBackend;
2236
2237    struct FailingSource;
2238
2239    impl WatchBackend for FailingBackend {
2240        fn create(
2241            &self,
2242            _targets: &WatchTargets,
2243            _wake: mpsc::Sender<()>,
2244        ) -> Result<Box<dyn WatchSource>, String> {
2245            Ok(Box::new(FailingSource))
2246        }
2247    }
2248
2249    impl WatchSource for FailingSource {
2250        fn watch(&mut self, directory: &Path) -> Result<(), String> {
2251            Err(format!(
2252                "cannot watch {}: the watch limit is reached",
2253                directory.display()
2254            ))
2255        }
2256    }
2257
2258    #[test]
2259    fn a_failed_registration_keeps_the_sweep() {
2260        let dir = tempfile::tempdir().unwrap();
2261        let db = dir.path().join("store.sqlite3");
2262        let (tick_tx, tick_rx) = mpsc::channel();
2263        let watch = start_with_backend(
2264            &db,
2265            "test-failed-registration",
2266            Duration::from_millis(20),
2267            move |tick| {
2268                let _ = tick_tx.send(tick);
2269                SqliteWatchControl::Handled
2270            },
2271            |_| Box::new(FailingBackend),
2272        )
2273        .expect("a failed notification registration leaves the watch on its sweep");
2274        assert_eq!(tick_rx.recv_timeout(HANG_GUARD), Ok(SqliteWatchTick::Sweep));
2275        drop(watch);
2276    }
2277
2278    /// Dropping a watch never waits for its notification source's teardown,
2279    /// which on macOS waits on the file-system event service; a watch started
2280    /// at once on the same database registers and sees a write meanwhile, and
2281    /// the stopped watch ticks no more.
2282    #[test]
2283    fn dropping_a_watch_never_waits_for_its_sources_teardown() {
2284        // It holds the process's unregister thread: alone in its process.
2285        alone_in_child_process(
2286            "dropping_a_watch_never_waits_for_its_sources_teardown",
2287            || {
2288                let dir = tempfile::tempdir().unwrap();
2289                let observed = test_support::observe_registrations_in(dir.path()).unwrap();
2290                let db = dir.path().join("store.sqlite3");
2291                log_row(
2292                    &db,
2293                    format_args!("dropping_a_watch_never_waits_for_its_sources_teardown starts"),
2294                );
2295                std::fs::write(&db, b"").unwrap();
2296                // The WAL exists before the watches start: the write below is
2297                // a data change only.
2298                let wal = sidecar_path(&db, "-wal");
2299                std::fs::write(&wal, b"").unwrap();
2300                let first_ticks = Arc::new(AtomicUsize::new(0));
2301                let first =
2302                    start_sqlite_change_watch(&db, "test-teardown-first", SQLITE_WATCH_SWEEP, {
2303                        let first_ticks = Arc::clone(&first_ticks);
2304                        move |_| {
2305                            first_ticks.fetch_add(1, Ordering::SeqCst);
2306                            SqliteWatchControl::Handled
2307                        }
2308                    })
2309                    .expect("watch starts");
2310                assert!(
2311                    observed.wait_registered(1, REGISTRATION_BOUND),
2312                    "the first registered: {:?}",
2313                    first.registration()
2314                );
2315                observed.hold_source_drops();
2316
2317                let (dropped_tx, dropped_rx) = mpsc::channel();
2318                let dropper = thread::spawn(move || {
2319                    drop(first);
2320                    let _ = dropped_tx.send(());
2321                });
2322                let returned = dropped_rx.recv_timeout(HANG_GUARD).is_ok();
2323                if !returned {
2324                    observed.release_source_drops();
2325                }
2326                dropper.join().unwrap();
2327                assert!(
2328                    returned,
2329                    "dropping the watch waited for its source's teardown"
2330                );
2331                let first_ticked = first_ticks.load(Ordering::SeqCst);
2332
2333                // A watch started at once on the same database works while the first
2334                // watch's source is still held.
2335                let (tick_tx, tick_rx) = mpsc::channel();
2336                let second =
2337                    start_sqlite_change_watch(&db, "test-teardown-second", SQLITE_WATCH_SWEEP, {
2338                        move |tick| {
2339                            let _ = tick_tx.send(tick);
2340                            SqliteWatchControl::Handled
2341                        }
2342                    })
2343                    .expect("watch starts");
2344                assert!(
2345                    observed.wait_registered(2, REGISTRATION_BOUND),
2346                    "the second registered: {:?}",
2347                    second.registration()
2348                );
2349                assert_eq!(
2350                    observed.source_drops_finished(),
2351                    0,
2352                    "the first source is held"
2353                );
2354                append(&wal, b"frame");
2355                next_changed(&tick_rx).expect("the second watch sees the write");
2356                // The held source still delivers that write, to the stopped watch
2357                // only, which ticks no more.
2358                assert_eq!(first_ticks.load(Ordering::SeqCst), first_ticked);
2359
2360                drop(second);
2361                observed.release_source_drops();
2362                assert!(
2363                    observed.wait_source_drops_finished(2, TEARDOWN_BOUND),
2364                    "both sources are dropped once released"
2365                );
2366            },
2367        );
2368    }
2369
2370    #[test]
2371    fn stopping_cancels_a_held_registration_and_keeps_its_thread() {
2372        let dir = tempfile::tempdir().unwrap();
2373        let held = test_support::hold_registrations_in(dir.path()).unwrap();
2374        let directory = dir.path().canonicalize().unwrap();
2375        let db = dir.path().join("store.sqlite3");
2376        let mut watch =
2377            start_sqlite_change_watch(&db, "test-held-stop", Duration::from_millis(20), |_| {
2378                SqliteWatchControl::Handled
2379            })
2380            .expect("watch starts");
2381        assert!(
2382            held.wait_entered(1, HANG_GUARD),
2383            "the registration reached its platform call"
2384        );
2385
2386        // The stop joins the worker and never waits for the registration.
2387        assert_eq!(watch.stop_and_join(), SqliteWatchJoin::Joined);
2388        assert_eq!(watch.registration(), SqliteWatchRegistration::Pending);
2389        assert_eq!(held.live_sources(), 1);
2390
2391        // Once the platform call returns, the registration discards its source
2392        // on its own thread instead of handing it to the stopped watch.
2393        held.release();
2394        assert_eq!(
2395            settled_registration(&watch),
2396            SqliteWatchRegistration::Cancelled
2397        );
2398        assert_eq!((held.live_sources(), held.dropped_sources()), (0, 1));
2399        // Its thread stayed with this watch, which joins it now.
2400        assert!(retained_in(&directory).is_empty());
2401        assert_eq!(watch.stop_and_join(), SqliteWatchJoin::Joined);
2402    }
2403
2404    #[test]
2405    fn dropping_a_watch_retains_its_held_registration_until_it_returns() {
2406        let dir = tempfile::tempdir().unwrap();
2407        let held = test_support::hold_registrations_in(dir.path()).unwrap();
2408        let directory = dir.path().canonicalize().unwrap();
2409        let db = dir.path().join("store.sqlite3");
2410        let watch =
2411            start_sqlite_change_watch(&db, "test-held-drop", Duration::from_millis(20), |_| {
2412                SqliteWatchControl::Handled
2413            })
2414            .expect("watch starts");
2415        assert!(
2416            held.wait_entered(1, HANG_GUARD),
2417            "the registration reached its platform call"
2418        );
2419
2420        // Dropping never waits for the registration and never detaches it:
2421        // the process keeps its thread, and reports it.
2422        drop(watch);
2423        let retained = retained_in(&directory);
2424        assert_eq!(retained.len(), 1, "{retained:?}");
2425        assert_eq!(retained[0].thread_name, "test-held-drop-register");
2426        assert_eq!(held.live_sources(), 1);
2427
2428        // Once the platform call returns, the registration discards its
2429        // source, and the process joins its thread.
2430        held.release();
2431        let deadline = Instant::now() + HANG_GUARD;
2432        while !retained_in(&directory).is_empty() && Instant::now() < deadline {
2433            thread::sleep(Duration::from_millis(2));
2434        }
2435        assert!(
2436            retained_in(&directory).is_empty(),
2437            "the process joins a retained registration once it returns"
2438        );
2439        assert_eq!((held.live_sources(), held.dropped_sources()), (0, 1));
2440    }
2441
2442    #[test]
2443    fn a_pending_registration_is_described_as_the_process_retains_it() {
2444        let dir = tempfile::tempdir().unwrap();
2445        let held = test_support::hold_registrations_in(dir.path()).unwrap();
2446        let directory = dir.path().canonicalize().unwrap();
2447        let db = dir.path().join("store.sqlite3");
2448        let watch =
2449            start_sqlite_change_watch(&db, "test-held-pending", Duration::from_millis(20), |_| {
2450                SqliteWatchControl::Handled
2451            })
2452            .expect("watch starts");
2453        assert!(
2454            held.wait_entered(1, HANG_GUARD),
2455            "the registration reached its platform call"
2456        );
2457
2458        // The stop cancels the held registration without ending it: it is
2459        // still pending, and dropping the watch hands exactly it to the
2460        // process.
2461        watch.request_stop();
2462        let pending = watch
2463            .pending_registration()
2464            .expect("a held registration is pending");
2465        assert_eq!(pending.directory, directory);
2466        assert_eq!(pending.thread_name, "test-held-pending-register");
2467        drop(watch);
2468        let retained = retained_in(&directory);
2469        assert_eq!(retained.len(), 1, "{retained:?}");
2470        assert_eq!(
2471            (&retained[0].directory, &retained[0].thread_name),
2472            (&pending.directory, &pending.thread_name)
2473        );
2474
2475        held.release();
2476        let deadline = Instant::now() + HANG_GUARD;
2477        while !retained_in(&directory).is_empty() && Instant::now() < deadline {
2478            thread::sleep(Duration::from_millis(2));
2479        }
2480        assert!(retained_in(&directory).is_empty());
2481    }
2482
2483    #[test]
2484    fn a_settled_registration_is_not_pending() {
2485        let dir = tempfile::tempdir().unwrap();
2486        // Released before the watch starts: the registration completes at
2487        // once, without a platform call.
2488        let held = test_support::hold_registrations_in(dir.path()).unwrap();
2489        held.release();
2490        let db = dir.path().join("store.sqlite3");
2491        let watch =
2492            start_sqlite_change_watch(&db, "test-settled-pending", SQLITE_WATCH_SWEEP, |_| {
2493                SqliteWatchControl::Handled
2494            })
2495            .expect("watch starts");
2496        assert_eq!(
2497            settled_registration(&watch),
2498            SqliteWatchRegistration::Registered
2499        );
2500        assert_eq!(watch.pending_registration(), None);
2501        watch.request_stop();
2502        assert_eq!(watch.pending_registration(), None);
2503    }
2504
2505    #[test]
2506    fn a_failed_registration_reports_its_reason() {
2507        let dir = tempfile::tempdir().unwrap();
2508        let db = dir.path().join("store.sqlite3");
2509        let watch = start_with_backend(
2510            &db,
2511            "test-failed-reason",
2512            SQLITE_WATCH_SWEEP,
2513            |_| SqliteWatchControl::Handled,
2514            |_| Box::new(FailingBackend),
2515        )
2516        .expect("watch starts");
2517        let registration = settled_registration(&watch);
2518        assert!(
2519            matches!(
2520                &registration,
2521                SqliteWatchRegistration::Failed(reason)
2522                    if reason.contains("the watch limit is reached")
2523            ),
2524            "{registration:?}"
2525        );
2526    }
2527
2528    /// Set in the child process of the panicking-registration test.
2529    const PANIC_CHILD: &str = "MEERKAT_TEST_SQLITE_WATCH_PANIC_CHILD";
2530    /// What that child prints once its watch has survived.
2531    const PANIC_CHILD_DONE: &str = "SQLITE_WATCH_PANIC_CONTAINED";
2532
2533    /// A source whose registration panics and whose drop panics again, as
2534    /// notify 8.2.0's inotify watcher does when its event-loop thread never
2535    /// started: dropping it while the first panic unwinds aborts the process.
2536    struct PanickingBackend;
2537
2538    struct PanickingSource;
2539
2540    impl WatchBackend for PanickingBackend {
2541        fn create(
2542            &self,
2543            _targets: &WatchTargets,
2544            _wake: mpsc::Sender<()>,
2545        ) -> Result<Box<dyn WatchSource>, String> {
2546            Ok(Box::new(PanickingSource))
2547        }
2548    }
2549
2550    #[allow(clippy::panic)]
2551    impl WatchSource for PanickingSource {
2552        fn watch(&mut self, _directory: &Path) -> Result<(), String> {
2553            panic!("the notification event loop is gone");
2554        }
2555    }
2556
2557    #[allow(clippy::panic)]
2558    impl Drop for PanickingSource {
2559        fn drop(&mut self) {
2560            panic!("the notification event loop is gone");
2561        }
2562    }
2563
2564    /// A registered source whose teardown waits and whose drop panics.
2565    struct PanicsOnDropBackend;
2566
2567    struct PanicsOnDropSource;
2568
2569    impl WatchBackend for PanicsOnDropBackend {
2570        fn create(
2571            &self,
2572            _targets: &WatchTargets,
2573            _wake: mpsc::Sender<()>,
2574        ) -> Result<Box<dyn WatchSource>, String> {
2575            Ok(Box::new(PanicsOnDropSource))
2576        }
2577    }
2578
2579    impl WatchSource for PanicsOnDropSource {
2580        fn watch(&mut self, _directory: &Path) -> Result<(), String> {
2581            Ok(())
2582        }
2583
2584        fn teardown_waits(&self) -> bool {
2585            true
2586        }
2587    }
2588
2589    #[allow(clippy::panic)]
2590    impl Drop for PanicsOnDropSource {
2591        fn drop(&mut self) {
2592            panic!("the run-loop thread did not shut down");
2593        }
2594    }
2595
2596    /// The unregister thread contains a panicking source drop: a source whose
2597    /// drop panics, queued between two others, leaves both dropped, and the
2598    /// thread still drops a source queued after them.
2599    #[test]
2600    fn a_panicking_source_drop_leaves_the_unregister_thread_running() {
2601        // It holds the process's unregister thread: alone in its process.
2602        alone_in_child_process(
2603            "a_panicking_source_drop_leaves_the_unregister_thread_running",
2604            || {
2605                let dir = tempfile::tempdir().unwrap();
2606                let observed = test_support::observe_registrations_in(dir.path()).unwrap();
2607                let db = dir.path().join("store.sqlite3");
2608                std::fs::write(&db, b"").unwrap();
2609                let observed_watch = |name: &str| {
2610                    start_sqlite_change_watch(&db, name, SQLITE_WATCH_SWEEP, |_| {
2611                        SqliteWatchControl::Handled
2612                    })
2613                    .expect("watch starts")
2614                };
2615                let first = observed_watch("test-unregister-first");
2616                assert!(
2617                    observed.wait_registered(1, REGISTRATION_BOUND),
2618                    "the first registered: {:?}",
2619                    first.registration()
2620                );
2621                let panicking = start_with_backend(
2622                    &dir.path().join("panics.sqlite3"),
2623                    "test-unregister-panics",
2624                    SQLITE_WATCH_SWEEP,
2625                    |_| SqliteWatchControl::Handled,
2626                    |_| Box::new(PanicsOnDropBackend),
2627                )
2628                .expect("watch starts");
2629                assert_eq!(
2630                    settled_registration(&panicking),
2631                    SqliteWatchRegistration::Registered
2632                );
2633                let second = observed_watch("test-unregister-second");
2634                assert!(
2635                    observed.wait_registered(2, REGISTRATION_BOUND),
2636                    "the second registered: {:?}",
2637                    second.registration()
2638                );
2639
2640                // Queued in this order behind the first source's held drop.
2641                observed.hold_source_drops();
2642                drop(first);
2643                drop(panicking);
2644                drop(second);
2645                observed.release_source_drops();
2646                assert!(
2647                    observed.wait_source_drops_finished(2, TEARDOWN_BOUND),
2648                    "both observed sources are dropped around the panicking one"
2649                );
2650
2651                // The thread keeps going: a source queued after the panic goes to it,
2652                // so its held drop does not hold the dropping thread, and it is
2653                // dropped once released.
2654                let third = observed_watch("test-unregister-third");
2655                assert!(
2656                    observed.wait_registered(3, REGISTRATION_BOUND),
2657                    "the third registered: {:?}",
2658                    third.registration()
2659                );
2660                observed.hold_source_drops();
2661                let (dropped_tx, dropped_rx) = mpsc::channel();
2662                let dropper = thread::spawn(move || {
2663                    drop(third);
2664                    let _ = dropped_tx.send(());
2665                });
2666                let returned = dropped_rx.recv_timeout(HANG_GUARD).is_ok();
2667                observed.release_source_drops();
2668                dropper.join().unwrap();
2669                assert!(
2670                    returned,
2671                    "a source queued after the panic went to the unregister thread"
2672                );
2673                assert!(
2674                    observed.wait_source_drops_finished(3, TEARDOWN_BOUND),
2675                    "the unregister thread drops a source queued after the panic"
2676                );
2677                let unregister = UNREGISTER
2678                    .get()
2679                    .and_then(Option::as_ref)
2680                    .expect("the unregister thread started");
2681                assert!(
2682                    !unregister._thread.is_finished(),
2683                    "the unregister thread runs"
2684                );
2685            },
2686        );
2687    }
2688
2689    /// The registration panics in a child process: a panic during a panic
2690    /// aborts the whole process, so only a child can show the watch survives.
2691    #[test]
2692    fn a_panicking_registration_leaves_the_sweep_running() {
2693        if std::env::var_os(PANIC_CHILD).is_some() {
2694            let dir = tempfile::tempdir().unwrap();
2695            let db = dir.path().join("store.sqlite3");
2696            let (tick_tx, tick_rx) = mpsc::channel();
2697            let watch = start_with_backend(
2698                &db,
2699                "test-panicking-registration",
2700                Duration::from_millis(20),
2701                move |tick| {
2702                    let _ = tick_tx.send(tick);
2703                    SqliteWatchControl::Handled
2704                },
2705                |_| Box::new(PanickingBackend),
2706            )
2707            .expect("watch starts");
2708            let registration = settled_registration(&watch);
2709            assert!(
2710                matches!(registration, SqliteWatchRegistration::Failed(_)),
2711                "{registration:?}"
2712            );
2713            assert_eq!(tick_rx.recv_timeout(HANG_GUARD), Ok(SqliteWatchTick::Sweep));
2714            drop(watch);
2715            println!("\n{PANIC_CHILD_DONE}");
2716            return;
2717        }
2718        #[allow(clippy::disallowed_methods)] // the test re-runs its own binary
2719        let output = std::process::Command::new(std::env::current_exe().unwrap())
2720            .args([
2721                "--exact",
2722                "watch::tests::a_panicking_registration_leaves_the_sweep_running",
2723                "--nocapture",
2724                "--test-threads=1",
2725            ])
2726            .env(PANIC_CHILD, "1")
2727            .output()
2728            .unwrap();
2729        let stdout = String::from_utf8_lossy(&output.stdout);
2730        assert!(
2731            output.status.success() && stdout.contains(PANIC_CHILD_DONE),
2732            "the watch did not survive its panicking registration ({}):\n{stdout}\n{}",
2733            output.status,
2734            String::from_utf8_lossy(&output.stderr)
2735        );
2736    }
2737
2738    /// Set in the child process that runs one test of this module alone.
2739    const ALONE_CHILD: &str = "MEERKAT_TEST_SQLITE_WATCH_ALONE_CHILD";
2740    /// What that child prints once the test passed.
2741    const ALONE_CHILD_DONE: &str = "SQLITE_WATCH_ALONE_TEST_PASSED";
2742
2743    /// Run `body`, the body of this module's test `test`, alone in a child
2744    /// process of this test binary, and require it to pass there. The
2745    /// registrations in flight are counted across the process, so a test
2746    /// that takes every place shares its process with no other watch test,
2747    /// whatever runs the tests: nextest's process per test, or plain `cargo
2748    /// test`'s concurrent threads in one process.
2749    fn alone_in_child_process(test: &str, body: impl FnOnce()) {
2750        if std::env::var_os(ALONE_CHILD).is_some() {
2751            body();
2752            println!("\n{ALONE_CHILD_DONE}");
2753            return;
2754        }
2755        #[allow(clippy::disallowed_methods)] // the test re-runs its own binary
2756        let output = std::process::Command::new(std::env::current_exe().unwrap())
2757            .args([
2758                "--exact",
2759                &format!("watch::tests::{test}"),
2760                "--nocapture",
2761                "--test-threads=1",
2762            ])
2763            .env(ALONE_CHILD, "1")
2764            .output()
2765            .unwrap();
2766        let stdout = String::from_utf8_lossy(&output.stdout);
2767        assert!(
2768            output.status.success() && stdout.contains(ALONE_CHILD_DONE),
2769            "{test} failed alone in its child process ({}):\n{stdout}\n{}",
2770            output.status,
2771            String::from_utf8_lossy(&output.stderr)
2772        );
2773    }
2774
2775    #[test]
2776    fn registrations_past_the_cap_leave_new_watches_on_their_sweep() {
2777        alone_in_child_process(
2778            "registrations_past_the_cap_leave_new_watches_on_their_sweep",
2779            || {
2780                // Alone in its process: no other registration holds a place.
2781                assert_eq!(REGISTRATIONS_IN_FLIGHT.load(Ordering::Acquire), 0);
2782                let dir = tempfile::tempdir().unwrap();
2783                let held = test_support::hold_registrations_in(dir.path()).unwrap();
2784                let watches: Vec<_> = (0..MAX_REGISTRATIONS_IN_FLIGHT)
2785                    .map(|index| {
2786                        start_sqlite_change_watch(
2787                            &dir.path().join(format!("store-{index}.sqlite3")),
2788                            "test-cap-held",
2789                            SQLITE_WATCH_SWEEP,
2790                            |_| SqliteWatchControl::Handled,
2791                        )
2792                        .expect("watch starts")
2793                    })
2794                    .collect();
2795                assert!(
2796                    held.wait_entered(MAX_REGISTRATIONS_IN_FLIGHT, HANG_GUARD),
2797                    "every registration under the cap reached its platform call"
2798                );
2799
2800                // One more watch starts with no registration thread, on its sweep
2801                // alone.
2802                let (tick_tx, tick_rx) = mpsc::channel();
2803                let over = start_sqlite_change_watch(
2804                    &dir.path().join("store-over.sqlite3"),
2805                    "test-cap-over",
2806                    Duration::from_millis(20),
2807                    move |tick| {
2808                        let _ = tick_tx.send(tick);
2809                        SqliteWatchControl::Handled
2810                    },
2811                )
2812                .expect("a watch past the cap still starts");
2813                let registration = over.registration();
2814                assert!(
2815                    matches!(
2816                        &registration,
2817                        SqliteWatchRegistration::Failed(reason) if reason.contains("already pending")
2818                    ),
2819                    "{registration:?}"
2820                );
2821                assert!(
2822                    over.registration.thread.is_none(),
2823                    "no registration thread started past the cap"
2824                );
2825                assert_eq!(held.live_sources(), MAX_REGISTRATIONS_IN_FLIGHT);
2826                assert_eq!(tick_rx.recv_timeout(HANG_GUARD), Ok(SqliteWatchTick::Sweep));
2827
2828                // Released, every held registration settles and gives its place back,
2829                // and a fresh watch registers again.
2830                held.release();
2831                for watch in &watches {
2832                    assert_eq!(
2833                        settled_registration(watch),
2834                        SqliteWatchRegistration::Registered
2835                    );
2836                }
2837                let deadline = Instant::now() + HANG_GUARD;
2838                while REGISTRATIONS_IN_FLIGHT.load(Ordering::Acquire) > 0
2839                    && Instant::now() < deadline
2840                {
2841                    thread::sleep(Duration::from_millis(2));
2842                }
2843                let fresh_dir = tempfile::tempdir().unwrap();
2844                let fresh = start_sqlite_change_watch(
2845                    &fresh_dir.path().join("store.sqlite3"),
2846                    "test-cap-fresh",
2847                    SQLITE_WATCH_SWEEP,
2848                    |_| SqliteWatchControl::Handled,
2849                )
2850                .expect("watch starts");
2851                assert_eq!(
2852                    settled_registration(&fresh),
2853                    SqliteWatchRegistration::Registered,
2854                    "a fresh watch registers once every place is given back"
2855                );
2856                drop(watches);
2857                drop(over);
2858            },
2859        );
2860    }
2861
2862    /// A burst of 64 registrations held at once, as a parallel startup holds
2863    /// them while FSEvents stalls, all reach their platform call: none is
2864    /// refused at the cap and left on its sweep for life. The burst is
2865    /// written out, not taken from the cap, so a lower cap fails here.
2866    #[test]
2867    fn a_held_burst_of_64_registrations_all_reach_their_platform_call() {
2868        alone_in_child_process(
2869            "a_held_burst_of_64_registrations_all_reach_their_platform_call",
2870            || {
2871                const BURST: usize = 64;
2872                // Alone in its process: no other registration holds a place.
2873                assert_eq!(REGISTRATIONS_IN_FLIGHT.load(Ordering::Acquire), 0);
2874                let dir = tempfile::tempdir().unwrap();
2875                let held = test_support::hold_registrations_in(dir.path()).unwrap();
2876                let watches: Vec<_> = (0..BURST)
2877                    .map(|index| {
2878                        start_sqlite_change_watch(
2879                            &dir.path().join(format!("store-{index}.sqlite3")),
2880                            "test-burst-held",
2881                            SQLITE_WATCH_SWEEP,
2882                            |_| SqliteWatchControl::Handled,
2883                        )
2884                        .expect("watch starts")
2885                    })
2886                    .collect();
2887                let refused = watches
2888                    .iter()
2889                    .filter(|watch| {
2890                        matches!(watch.registration(), SqliteWatchRegistration::Failed(_))
2891                    })
2892                    .count();
2893                assert_eq!(refused, 0, "registrations of the burst refused at the cap");
2894                assert!(
2895                    held.wait_entered(BURST, HANG_GUARD),
2896                    "every registration of the burst reached its platform call"
2897                );
2898                held.release();
2899                for watch in &watches {
2900                    assert_eq!(
2901                        settled_registration(watch),
2902                        SqliteWatchRegistration::Registered
2903                    );
2904                }
2905            },
2906        );
2907    }
2908
2909    /// Every WARN event this test process emitted, with the name of the
2910    /// thread that emitted it.
2911    static WARNINGS: Mutex<Vec<(Option<String>, String)>> = Mutex::new(Vec::new());
2912
2913    /// A process-wide subscriber that records WARN events into [`WARNINGS`]: a
2914    /// registration settles on its own thread, which a thread-local
2915    /// subscriber would never see.
2916    struct WarningCapture;
2917
2918    struct MessageField(String);
2919
2920    impl tracing::field::Visit for MessageField {
2921        fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
2922            if field.name() == "message" {
2923                self.0 = format!("{value:?}");
2924            }
2925        }
2926    }
2927
2928    impl tracing::Subscriber for WarningCapture {
2929        fn enabled(&self, metadata: &tracing::Metadata<'_>) -> bool {
2930            *metadata.level() == tracing::Level::WARN
2931        }
2932
2933        fn new_span(&self, _span: &tracing::span::Attributes<'_>) -> tracing::span::Id {
2934            tracing::span::Id::from_u64(1)
2935        }
2936
2937        fn record(&self, _span: &tracing::span::Id, _values: &tracing::span::Record<'_>) {}
2938
2939        fn record_follows_from(&self, _span: &tracing::span::Id, _follows: &tracing::span::Id) {}
2940
2941        fn event(&self, event: &tracing::Event<'_>) {
2942            let mut message = MessageField(String::new());
2943            event.record(&mut message);
2944            lock(&WARNINGS).push((thread::current().name().map(str::to_string), message.0));
2945        }
2946
2947        fn enter(&self, _span: &tracing::span::Id) {}
2948
2949        fn exit(&self, _span: &tracing::span::Id) {}
2950    }
2951
2952    /// Install [`WarningCapture`] as this process's global subscriber, once.
2953    fn capture_warnings() {
2954        static INSTALLED: OnceLock<bool> = OnceLock::new();
2955        let installed = *INSTALLED
2956            .get_or_init(|| tracing::subscriber::set_global_default(WarningCapture).is_ok());
2957        assert!(
2958            installed,
2959            "the warning capture is this process's global subscriber"
2960        );
2961    }
2962
2963    fn warnings_from(thread_name: &str) -> Vec<String> {
2964        lock(&WARNINGS)
2965            .iter()
2966            .filter(|(name, _)| name.as_deref() == Some(thread_name))
2967            .map(|(_, message)| message.clone())
2968            .collect()
2969    }
2970
2971    /// A source whose registration waits for its release, then fails, as an
2972    /// inotify watch past the user's limit does.
2973    struct GatedFailingBackend {
2974        entered: mpsc::Sender<()>,
2975        release: Mutex<Option<mpsc::Receiver<()>>>,
2976    }
2977
2978    struct GatedFailingSource {
2979        entered: mpsc::Sender<()>,
2980        release: mpsc::Receiver<()>,
2981    }
2982
2983    impl WatchBackend for GatedFailingBackend {
2984        fn create(
2985            &self,
2986            _targets: &WatchTargets,
2987            _wake: mpsc::Sender<()>,
2988        ) -> Result<Box<dyn WatchSource>, String> {
2989            let release = lock(&self.release)
2990                .take()
2991                .ok_or_else(|| "created twice".to_string())?;
2992            Ok(Box::new(GatedFailingSource {
2993                entered: self.entered.clone(),
2994                release,
2995            }))
2996        }
2997    }
2998
2999    impl WatchSource for GatedFailingSource {
3000        fn watch(&mut self, directory: &Path) -> Result<(), String> {
3001            let _ = self.entered.send(());
3002            let _ = self.release.recv();
3003            Err(format!(
3004                "cannot watch {}: the watch limit is reached",
3005                directory.display()
3006            ))
3007        }
3008    }
3009
3010    /// Start a watch whose registration fails once released. Returns the
3011    /// watch, the signal that its registration entered, and its release.
3012    fn start_failing_after_release(
3013        dir: &Path,
3014        name: &str,
3015    ) -> (SqliteChangeWatch, mpsc::Receiver<()>, mpsc::Sender<()>) {
3016        let (entered_tx, entered_rx) = mpsc::channel();
3017        let (release_tx, release_rx) = mpsc::channel();
3018        let backend = GatedFailingBackend {
3019            entered: entered_tx,
3020            release: Mutex::new(Some(release_rx)),
3021        };
3022        let watch = start_with_backend(
3023            &dir.join("store.sqlite3"),
3024            name,
3025            SQLITE_WATCH_SWEEP,
3026            |_| SqliteWatchControl::Handled,
3027            move |_| Box::new(backend),
3028        )
3029        .expect("watch starts");
3030        (watch, entered_rx, release_tx)
3031    }
3032
3033    #[test]
3034    fn a_registration_failing_after_its_watch_stopped_is_cancelled_quietly() {
3035        capture_warnings();
3036        // A live watch whose registration fails reports it and warns once:
3037        // the capture sees registration threads.
3038        let live_dir = tempfile::tempdir().unwrap();
3039        let (live, live_entered, live_release) =
3040            start_failing_after_release(live_dir.path(), "test-fail-live");
3041        live_entered
3042            .recv_timeout(HANG_GUARD)
3043            .expect("the live registration reached its platform call");
3044        live_release.send(()).unwrap();
3045        assert!(matches!(
3046            settled_registration(&live),
3047            SqliteWatchRegistration::Failed(_)
3048        ));
3049        assert_eq!(warnings_from("test-fail-live-register").len(), 1);
3050
3051        // A watch that stopped first is gone: its failed registration is
3052        // cancelled, with no warning about a sweep nobody runs any more.
3053        let stopped_dir = tempfile::tempdir().unwrap();
3054        let (mut stopped, stopped_entered, stopped_release) =
3055            start_failing_after_release(stopped_dir.path(), "test-fail-stopped");
3056        stopped_entered
3057            .recv_timeout(HANG_GUARD)
3058            .expect("the registration reached its platform call");
3059        assert_eq!(stopped.stop_and_join(), SqliteWatchJoin::Joined);
3060        stopped_release.send(()).unwrap();
3061        assert_eq!(
3062            (
3063                settled_registration(&stopped),
3064                warnings_from("test-fail-stopped-register")
3065            ),
3066            (SqliteWatchRegistration::Cancelled, Vec::<String>::new())
3067        );
3068    }
3069}