Skip to main content

postgres_test_harness/
harness.rs

1use std::{
2    collections::VecDeque,
3    future::Future,
4    sync::{
5        Arc, Mutex, Weak,
6        atomic::{AtomicUsize, Ordering},
7    },
8    task::Poll,
9    time::{Duration, SystemTime},
10};
11
12use tokio::sync::{OwnedSemaphorePermit, Semaphore};
13
14use crate::{
15    BoxError, ConnectionLimits, Error, HarnessConfig, ProjectName, Result, TemplateFingerprint,
16    TemplateSpec,
17    admin::{
18        AdminClient, AdminDatabaseUrl, AdminSessionDisposition, DatabaseRecord, PersistentClient,
19        acquire_shared_template_advisory_lock, acquire_template_advisory_lock, advisory_key,
20        connect_admin, disable_database_connections, drop_database, find_database, list_databases,
21        release_advisory_lock, release_shared_advisory_lock, set_database_metadata,
22        terminate_database_connections, try_acquire_advisory_lock, validate_postgres_18,
23    },
24    admission::ManagedDatabaseCreationFailure,
25    cleanup::{CleanupOutcome, log_cleanup},
26    metadata::{ResourceMetadata, TemplateState},
27    name::{DatabaseKind, DatabaseName},
28    server::{ServerInner, TemplateCacheAction, TemplateInner, run_blocking},
29};
30
31const DEFAULT_CLEANUP_OPERATION_TIMEOUT: Duration = Duration::from_secs(90);
32
33/// Process-local owner of a PostgreSQL 18 server and its database capacity.
34#[derive(Clone)]
35pub struct PostgresHarness {
36    server: Arc<ServerInner>,
37}
38
39impl PostgresHarness {
40    pub async fn start(config: HarnessConfig) -> Result<Self> {
41        let server = ServerInner::start(config).await?;
42        let harness = Self { server };
43        if harness.server.is_external() && harness.server.cleanup_on_start {
44            cleanup_stale_with(
45                harness.server.admin_url.clone(),
46                harness.server.project.clone(),
47                harness.server.stale_after,
48                harness.server.operation_timeout,
49            )
50            .await?;
51        }
52        Ok(harness)
53    }
54
55    pub fn project(&self) -> &ProjectName {
56        &self.server.project
57    }
58
59    pub fn admin_database_url(&self) -> &str {
60        self.server.admin_url.as_str()
61    }
62
63    pub fn is_external(&self) -> bool {
64        self.server.is_external()
65    }
66
67    /// Returns the owned container ID, or `None` in external-server mode.
68    ///
69    /// Builds without the `containers` feature always return `None`.
70    pub fn container_id(&self) -> Option<&str> {
71        self.server.container_id()
72    }
73
74    /// Returns the resolved downstream connection-permit policy used by this
75    /// harness's database admission control.
76    pub fn connection_limits(&self) -> ConnectionLimits {
77        self.server.connection_limits
78    }
79
80    /// Closes database admission, drains accepted cleanup, and removes an owned
81    /// container.
82    ///
83    /// Admission remains closed after the first owned shutdown attempt, even
84    /// if container removal fails. External servers and their admission remain
85    /// untouched. Repeated and concurrent calls do not retry removal; the call
86    /// that completes the terminal worker reports any removal error, while
87    /// later calls observe completed shutdown. Active leases remain owned but
88    /// can no longer contact a successfully removed server. Cleanup submissions
89    /// already accepted by the bounded server queue finish before its pooled
90    /// admin sessions and container are closed. Deferred failures are returned
91    /// from this barrier. External-server shutdown remains a no-op, including
92    /// in builds without the `containers` feature; call
93    /// [`Self::drain_deferred_cleanup`] explicitly for an external server.
94    pub async fn shutdown(&self) -> Result<()> {
95        self.server.shutdown_container().await
96    }
97
98    /// Waits for every database cleanup accepted before this call.
99    ///
100    /// Failures from [`DatabaseLease::defer_cleanup`], the `Drop` fallback, or
101    /// an awaited cleanup whose caller was cancelled are retained and returned
102    /// exactly once. Cleanup submitted concurrently after the barrier begins is
103    /// covered by a later drain. A failure that may leave a residual database
104    /// closes new admission so storage cannot accumulate without bound.
105    pub async fn drain_deferred_cleanup(&self) -> Result<()> {
106        let database_cleanup = self.server.database_cleanup.clone();
107        let outcome = run_blocking(move || Ok(database_cleanup.begin_drain())).await?;
108        outcome.finish()
109    }
110
111    /// Creates a pristine database from PostgreSQL's built-in `template0`.
112    pub async fn empty_database(&self) -> Result<DatabaseLease> {
113        create_test_database(self.server.clone(), "template0").await
114    }
115
116    /// Gets or initializes one immutable, content-addressed template database.
117    ///
118    /// Calls with the same fingerprint on clones of this harness single-flight
119    /// initialization and share one live template handle. The cache is weak:
120    /// dropping the final [`DatabaseTemplate`] releases its retained PostgreSQL
121    /// shared-lock session. Separate harness starts continue to coordinate through
122    /// PostgreSQL advisory locks.
123    pub async fn template<F, Fut>(
124        &self,
125        spec: TemplateSpec,
126        initializer: F,
127    ) -> Result<DatabaseTemplate>
128    where
129        F: FnOnce(String) -> Fut,
130        Fut: Future<Output = std::result::Result<(), BoxError>>,
131    {
132        let fingerprint = spec.fingerprint();
133        let mut initializer = Some(initializer);
134        loop {
135            match self.server.template_cache_action(fingerprint) {
136                TemplateCacheAction::Ready(inner) => return Ok(DatabaseTemplate { inner }),
137                TemplateCacheAction::Wait(waiter) => {
138                    if let Some(inner) = waiter.wait().await {
139                        return Ok(DatabaseTemplate { inner });
140                    }
141                }
142                TemplateCacheAction::Initialize(cache_initializer) => {
143                    let name = DatabaseName::template(&self.server.project, fingerprint);
144                    let server = self.server.clone();
145                    let preparation =
146                        run_blocking(move || begin_template(server, name, fingerprint)).await?;
147
148                    let template_lock = match preparation {
149                        TemplatePreparation::Ready(template_lock) => template_lock,
150                        TemplatePreparation::Initialize(initialization) => {
151                            let database_url = initialization
152                                .server
153                                .admin_url
154                                .database_url(&initialization.name);
155                            let initialize = initializer
156                                .take()
157                                .expect("a template caller initializes at most once");
158                            match initialize(database_url).await {
159                                Ok(()) => run_blocking(move || initialization.finish()).await?,
160                                Err(initializer) => {
161                                    let cleanup =
162                                        run_blocking(move || initialization.abort()).await;
163                                    return match cleanup {
164                                        Ok(()) => Err(Error::TemplateInitializer {
165                                            source: initializer,
166                                        }),
167                                        Err(cleanup) => Err(Error::TemplateInitializerAndCleanup {
168                                            initializer,
169                                            cleanup: Box::new(cleanup),
170                                        }),
171                                    };
172                                }
173                            }
174                        }
175                    };
176
177                    let inner = Arc::new(TemplateInner::new(
178                        self.server.clone(),
179                        DatabaseName::template(&self.server.project, fingerprint),
180                        fingerprint,
181                        template_lock,
182                    ));
183                    return Ok(DatabaseTemplate {
184                        inner: cache_initializer.publish(inner),
185                    });
186                }
187            }
188        }
189    }
190}
191
192/// Immutable initialized database that can be cloned for individual tests.
193#[derive(Clone)]
194pub struct DatabaseTemplate {
195    inner: Arc<TemplateInner>,
196}
197
198impl DatabaseTemplate {
199    pub fn database_name(&self) -> &str {
200        self.inner.name().as_str()
201    }
202
203    pub fn fingerprint(&self) -> TemplateFingerprint {
204        self.inner.fingerprint()
205    }
206
207    pub async fn database(&self) -> Result<DatabaseLease> {
208        create_test_database(self.inner.server().clone(), self.inner.name().as_str()).await
209    }
210
211    /// Creates and fills an opt-in bounded queue of pristine disposable databases.
212    ///
213    /// Idle databases retain this template's shared lock but consume no
214    /// downstream connection permits. The capacity may not exceed the
215    /// harness's effective simultaneous-lease limit. Every leased database is
216    /// dropped after use; the background lifecycle queue creates a distinct
217    /// replacement rather than reusing dirty state.
218    pub async fn prewarm(&self, capacity: usize) -> Result<PrewarmedDatabasePool> {
219        let max_capacity = self
220            .inner
221            .server()
222            .connection_limits
223            .max_simultaneous_leases();
224        if capacity == 0 || capacity > max_capacity {
225            return Err(Error::InvalidPrewarmCapacity {
226                capacity,
227                max_capacity,
228            });
229        }
230
231        let pool = PrewarmedDatabasePool::new(self.inner.clone(), capacity)?;
232        for _ in 0..capacity {
233            let creation = pool.inner.begin_creation()?;
234            let prepared = create_unpublished_database(
235                self.inner.server().clone(),
236                self.inner.name().as_str().to_owned(),
237            )
238            .await;
239            match prepared {
240                Ok(prepared) => creation.publish(prepared),
241                Err(error) => {
242                    creation.fail();
243                    return Err(error);
244                }
245            }
246        }
247        Ok(pool)
248    }
249}
250
251/// Current resource state of a bounded prewarmed database queue.
252#[derive(Clone, Copy, Debug, Eq, PartialEq)]
253#[non_exhaustive]
254pub struct PrewarmPoolStatus {
255    capacity: usize,
256    ready: usize,
257    leased: usize,
258    creating: usize,
259    deleting: usize,
260    closed: bool,
261}
262
263impl PrewarmPoolStatus {
264    pub const fn capacity(&self) -> usize {
265        self.capacity
266    }
267
268    pub const fn ready(&self) -> usize {
269        self.ready
270    }
271
272    pub const fn leased(&self) -> usize {
273        self.leased
274    }
275
276    pub const fn creating(&self) -> usize {
277        self.creating
278    }
279
280    pub const fn deleting(&self) -> usize {
281        self.deleting
282    }
283
284    pub const fn is_closed(&self) -> bool {
285        self.closed
286    }
287
288    /// Returns the number of capacity slots currently represented by ready,
289    /// leased, creating, or deleting work.
290    pub const fn occupied_slots(&self) -> usize {
291        self.ready + self.leased + self.creating + self.deleting
292    }
293}
294
295/// Opt-in bounded queue of never-used disposable databases cloned from one template.
296pub struct PrewarmedDatabasePool {
297    inner: Arc<PrewarmPoolInner>,
298}
299
300impl PrewarmedDatabasePool {
301    fn new(template: Arc<TemplateInner>, capacity: usize) -> Result<Self> {
302        let inner = Arc::new(PrewarmPoolInner {
303            template,
304            capacity,
305            public_handles: AtomicUsize::new(1),
306            ready_admission: Arc::new(Semaphore::new(0)),
307            closed: Arc::new(Semaphore::new(0)),
308            state: Mutex::new(PrewarmPoolState::new()),
309        });
310        inner.template.server().register_prewarm_pool(&inner)?;
311        Ok(Self { inner })
312    }
313
314    /// Returns the configured database-slot bound.
315    pub fn capacity(&self) -> usize {
316        self.inner.capacity
317    }
318
319    /// Returns a point-in-time view of the bounded queue state.
320    pub fn status(&self) -> PrewarmPoolStatus {
321        self.inner.status()
322    }
323
324    /// Leases one never-used database from the ready queue.
325    ///
326    /// A ready token is reserved before downstream application capacity. If
327    /// application admission or this pool is closed, or if this future is
328    /// cancelled, the token is returned and the idle database remains in the
329    /// queue until pool cleanup consumes it.
330    pub async fn database(&self) -> Result<DatabaseLease> {
331        let ready = self
332            .inner
333            .ready_admission
334            .clone()
335            .acquire_owned()
336            .await
337            .map_err(|_| Error::PrewarmPoolClosed)?;
338        let permit = acquire_database_permit_or_pool_close(
339            self.inner.template.server().acquire_database_permit(),
340            self.inner.closed.clone(),
341        )
342        .await?;
343        let Some(prepared) = self.inner.take_ready() else {
344            return Err(Error::PrewarmPoolClosed);
345        };
346        ready.forget();
347        Ok(prepared.into_lease(permit, Arc::downgrade(&self.inner)))
348    }
349
350    /// Closes this queue, drops all idle databases, and waits for lifecycle
351    /// work accepted before the resulting server-scoped cleanup barrier.
352    ///
353    /// Leases still owned by callers are not revoked. They remain responsible
354    /// for their own cleanup and will not trigger a refill after this call.
355    pub async fn shutdown(self) -> Result<()> {
356        self.inner.close_and_queue_ready();
357        let database_cleanup = self.inner.template.server().database_cleanup.clone();
358        let outcome = run_blocking(move || Ok(database_cleanup.begin_drain())).await?;
359        outcome.finish()
360    }
361}
362
363impl Clone for PrewarmedDatabasePool {
364    fn clone(&self) -> Self {
365        self.inner.public_handles.fetch_add(1, Ordering::Relaxed);
366        Self {
367            inner: self.inner.clone(),
368        }
369    }
370}
371
372impl std::fmt::Debug for PrewarmedDatabasePool {
373    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
374        formatter
375            .debug_struct("PrewarmedDatabasePool")
376            .field("template", &self.inner.template.name().as_str())
377            .field("status", &self.status())
378            .finish()
379    }
380}
381
382impl Drop for PrewarmedDatabasePool {
383    fn drop(&mut self) {
384        if self.inner.public_handles.fetch_sub(1, Ordering::AcqRel) == 1 {
385            self.inner.close_and_queue_ready();
386        }
387    }
388}
389
390pub(crate) struct PrewarmPoolInner {
391    template: Arc<TemplateInner>,
392    capacity: usize,
393    public_handles: AtomicUsize,
394    ready_admission: Arc<Semaphore>,
395    closed: Arc<Semaphore>,
396    state: Mutex<PrewarmPoolState>,
397}
398
399struct PrewarmPoolState {
400    ready: VecDeque<PreparedDatabase>,
401    leased: usize,
402    creating: usize,
403    deleting: usize,
404    accepting: bool,
405    cleanup_started: bool,
406}
407
408impl PrewarmPoolState {
409    fn new() -> Self {
410        Self {
411            ready: VecDeque::new(),
412            leased: 0,
413            creating: 0,
414            deleting: 0,
415            accepting: true,
416            cleanup_started: false,
417        }
418    }
419
420    fn occupied_slots(&self) -> usize {
421        self.ready.len() + self.leased + self.creating + self.deleting
422    }
423
424    fn fail_and_begin_cleanup(
425        &mut self,
426        phase: PrewarmReturnPhase,
427        capacity: usize,
428    ) -> Option<VecDeque<PreparedDatabase>> {
429        match phase {
430            PrewarmReturnPhase::Deleting => {
431                debug_assert!(self.deleting > 0);
432                self.deleting = self.deleting.saturating_sub(1);
433            }
434            PrewarmReturnPhase::Creating => {
435                debug_assert!(self.creating > 0);
436                self.creating = self.creating.saturating_sub(1);
437            }
438            PrewarmReturnPhase::Complete => return None,
439        }
440        self.begin_cleanup(capacity)
441    }
442
443    fn begin_cleanup(&mut self, capacity: usize) -> Option<VecDeque<PreparedDatabase>> {
444        self.accepting = false;
445        if self.cleanup_started {
446            return None;
447        }
448        self.cleanup_started = true;
449        let ready = std::mem::take(&mut self.ready);
450        self.deleting += ready.len();
451        debug_assert!(self.occupied_slots() <= capacity);
452        Some(ready)
453    }
454}
455
456impl PrewarmPoolInner {
457    fn lock_state(&self) -> std::sync::MutexGuard<'_, PrewarmPoolState> {
458        self.state
459            .lock()
460            .unwrap_or_else(std::sync::PoisonError::into_inner)
461    }
462
463    fn status(&self) -> PrewarmPoolStatus {
464        let state = self.lock_state();
465        PrewarmPoolStatus {
466            capacity: self.capacity,
467            ready: state.ready.len(),
468            leased: state.leased,
469            creating: state.creating,
470            deleting: state.deleting,
471            closed: !state.accepting,
472        }
473    }
474
475    fn begin_creation(self: &Arc<Self>) -> Result<PrewarmCreation> {
476        let mut state = self.lock_state();
477        if !state.accepting {
478            return Err(Error::PrewarmPoolClosed);
479        }
480        debug_assert!(state.occupied_slots() < self.capacity);
481        if state.occupied_slots() >= self.capacity {
482            return Err(Error::PrewarmPoolClosed);
483        }
484        state.creating += 1;
485        drop(state);
486        Ok(PrewarmCreation {
487            pool: self.clone(),
488            active: true,
489        })
490    }
491
492    fn take_ready(&self) -> Option<PreparedDatabase> {
493        let mut state = self.lock_state();
494        if !state.accepting {
495            return None;
496        }
497        let prepared = state.ready.pop_front()?;
498        state.leased += 1;
499        debug_assert!(state.occupied_slots() <= self.capacity);
500        Some(prepared)
501    }
502
503    fn begin_return(self: &Arc<Self>) -> PrewarmReturn {
504        let mut state = self.lock_state();
505        debug_assert!(state.leased > 0);
506        state.leased = state.leased.saturating_sub(1);
507        state.deleting += 1;
508        debug_assert!(state.occupied_slots() <= self.capacity);
509        drop(state);
510        PrewarmReturn {
511            pool: self.clone(),
512            phase: PrewarmReturnPhase::Deleting,
513        }
514    }
515
516    fn finish_creation(
517        self: &Arc<Self>,
518        prepared: PreparedDatabase,
519    ) -> Option<(PreparedDatabase, ClosedPrewarmDeletion)> {
520        let mut state = self.lock_state();
521        debug_assert!(state.creating > 0);
522        state.creating = state.creating.saturating_sub(1);
523        if state.accepting && !state.cleanup_started {
524            state.ready.push_back(prepared);
525            debug_assert!(state.occupied_slots() <= self.capacity);
526            drop(state);
527            self.ready_admission.add_permits(1);
528            None
529        } else {
530            state.deleting += 1;
531            debug_assert!(state.occupied_slots() <= self.capacity);
532            Some((
533                prepared,
534                ClosedPrewarmDeletion {
535                    pool: self.clone(),
536                    active: true,
537                },
538            ))
539        }
540    }
541
542    fn fail_phase(self: &Arc<Self>, phase: PrewarmReturnPhase) {
543        let ready = self
544            .lock_state()
545            .fail_and_begin_cleanup(phase, self.capacity);
546        self.finish_close(ready);
547    }
548
549    pub(crate) fn close_and_queue_ready(self: &Arc<Self>) {
550        let ready = self.lock_state().begin_cleanup(self.capacity);
551        self.finish_close(ready);
552    }
553
554    fn finish_close(self: &Arc<Self>, ready: Option<VecDeque<PreparedDatabase>>) {
555        self.ready_admission.close();
556        self.closed.close();
557        let Some(ready) = ready else {
558            return;
559        };
560        for prepared in ready {
561            queue_closed_prepared_database(self.clone(), prepared);
562        }
563    }
564}
565
566async fn acquire_database_permit_or_pool_close<F>(
567    database_admission: F,
568    pool_closed: Arc<Semaphore>,
569) -> Result<OwnedSemaphorePermit>
570where
571    F: Future<Output = Result<OwnedSemaphorePermit>>,
572{
573    let mut database_admission = std::pin::pin!(database_admission);
574    let mut pool_closed = std::pin::pin!(pool_closed.acquire_owned());
575    std::future::poll_fn(|context| {
576        if let Poll::Ready(result) = database_admission.as_mut().poll(context) {
577            return Poll::Ready(result);
578        }
579        if pool_closed.as_mut().poll(context).is_ready() {
580            return Poll::Ready(Err(Error::PrewarmPoolClosed));
581        }
582        Poll::Pending
583    })
584    .await
585}
586
587struct PreparedDatabase {
588    name: DatabaseName,
589    database_url: String,
590}
591
592impl PreparedDatabase {
593    fn into_lease(
594        self,
595        permit: OwnedSemaphorePermit,
596        prewarm_pool: Weak<PrewarmPoolInner>,
597    ) -> DatabaseLease {
598        DatabaseLease {
599            inner: Some(DatabaseLeaseInner {
600                server: prewarm_pool
601                    .upgrade()
602                    .expect("a leased prewarmed database retains its public pool")
603                    .template
604                    .server()
605                    .clone(),
606                name: self.name,
607                permit,
608                prewarm_pool: Some(prewarm_pool),
609            }),
610            database_url: self.database_url,
611        }
612    }
613}
614
615struct UnpublishedDatabase {
616    server: Arc<ServerInner>,
617    prepared: Option<PreparedDatabase>,
618}
619
620impl UnpublishedDatabase {
621    fn publish(mut self) -> PreparedDatabase {
622        self.prepared
623            .take()
624            .expect("an unpublished database is published at most once")
625    }
626}
627
628impl Drop for UnpublishedDatabase {
629    fn drop(&mut self) {
630        if let Some(prepared) = self.prepared.take() {
631            queue_unpublished_database(self.server.clone(), prepared);
632        }
633    }
634}
635
636struct PrewarmCreation {
637    pool: Arc<PrewarmPoolInner>,
638    active: bool,
639}
640
641impl PrewarmCreation {
642    fn publish(mut self, unpublished: UnpublishedDatabase) {
643        let prepared = unpublished.publish();
644        if let Some((prepared, deletion)) = self.pool.finish_creation(prepared) {
645            queue_counted_prepared_database(
646                self.pool.template.server().clone(),
647                prepared,
648                deletion,
649            );
650        }
651        self.active = false;
652    }
653
654    fn fail(mut self) {
655        self.pool.fail_phase(PrewarmReturnPhase::Creating);
656        self.active = false;
657    }
658}
659
660impl Drop for PrewarmCreation {
661    fn drop(&mut self) {
662        if self.active {
663            self.pool.fail_phase(PrewarmReturnPhase::Creating);
664        }
665    }
666}
667
668#[derive(Clone, Copy, Debug, Eq, PartialEq)]
669enum PrewarmReturnPhase {
670    Deleting,
671    Creating,
672    Complete,
673}
674
675struct PrewarmReturn {
676    pool: Arc<PrewarmPoolInner>,
677    phase: PrewarmReturnPhase,
678}
679
680impl PrewarmReturn {
681    fn after_drop(&mut self) -> bool {
682        let mut state = self.pool.lock_state();
683        debug_assert_eq!(self.phase, PrewarmReturnPhase::Deleting);
684        debug_assert!(state.deleting > 0);
685        state.deleting = state.deleting.saturating_sub(1);
686        if state.accepting && !state.cleanup_started {
687            state.creating += 1;
688            self.phase = PrewarmReturnPhase::Creating;
689            debug_assert!(state.occupied_slots() <= self.pool.capacity);
690            true
691        } else {
692            self.phase = PrewarmReturnPhase::Complete;
693            false
694        }
695    }
696
697    fn publish(
698        &mut self,
699        prepared: PreparedDatabase,
700    ) -> Option<(PreparedDatabase, ClosedPrewarmDeletion)> {
701        debug_assert_eq!(self.phase, PrewarmReturnPhase::Creating);
702        let unpublished = self.pool.finish_creation(prepared);
703        self.phase = PrewarmReturnPhase::Complete;
704        unpublished
705    }
706}
707
708impl Drop for PrewarmReturn {
709    fn drop(&mut self) {
710        if self.phase != PrewarmReturnPhase::Complete {
711            self.pool.fail_phase(self.phase);
712            self.phase = PrewarmReturnPhase::Complete;
713        }
714    }
715}
716
717struct ClosedPrewarmDeletion {
718    pool: Arc<PrewarmPoolInner>,
719    active: bool,
720}
721
722impl ClosedPrewarmDeletion {
723    fn finish(mut self) {
724        let mut state = self.pool.lock_state();
725        debug_assert!(state.deleting > 0);
726        state.deleting = state.deleting.saturating_sub(1);
727        self.active = false;
728    }
729}
730
731impl Drop for ClosedPrewarmDeletion {
732    fn drop(&mut self) {
733        if self.active {
734            let mut state = self.pool.lock_state();
735            debug_assert!(state.deleting > 0);
736            state.deleting = state.deleting.saturating_sub(1);
737        }
738    }
739}
740
741/// Exclusive ownership of one disposable test database.
742///
743/// Dropping a lease enqueues cleanup without waiting for queue capacity. The
744/// fallback retains its connection-budget permit until cleanup finishes, so
745/// saturation delays later database acquisition instead of blocking `Drop`.
746pub struct DatabaseLease {
747    inner: Option<DatabaseLeaseInner>,
748    database_url: String,
749}
750
751impl DatabaseLease {
752    pub fn database_name(&self) -> &str {
753        self.inner
754            .as_ref()
755            .expect("database lease is present until cleanup consumes it")
756            .name
757            .as_str()
758    }
759
760    pub fn database_url(&self) -> &str {
761        &self.database_url
762    }
763
764    /// Drops this database and waits for confirmation.
765    ///
766    /// The database's connection-budget permit remains held until cleanup
767    /// finishes. A failed database drop closes new database admission for this
768    /// harness so residual storage remains bounded. Close all application
769    /// connections and pools before calling.
770    pub async fn cleanup(mut self) -> Result<()> {
771        let inner = self
772            .inner
773            .take()
774            .expect("database lease cleanup runs at most once");
775        let completion = run_blocking(move || inner.queue_awaited_cleanup()).await?;
776        completion
777            .await
778            .map_err(|_| Error::CleanupWorkerStopped)?
779            .finish()
780    }
781
782    /// Returns this database to the bounded server-scoped cleanup queue.
783    ///
784    /// This future completes once the queue accepts the database, not when its
785    /// `DROP` finishes. It applies backpressure while the bounded queue is full,
786    /// then releases the database's connection-budget permit. Call
787    /// [`PostgresHarness::drain_deferred_cleanup`] before process or external
788    /// server teardown to await final completion and observe cleanup failures.
789    /// A failure that may leave a residual database closes new database
790    /// admission for this harness. Close all application connections and pools
791    /// before calling.
792    pub async fn defer_cleanup(mut self) -> Result<()> {
793        let inner = self
794            .inner
795            .take()
796            .expect("database lease cleanup runs at most once");
797        run_blocking(move || inner.queue_deferred_cleanup()).await
798    }
799}
800
801impl std::fmt::Debug for DatabaseLease {
802    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
803        formatter
804            .debug_struct("DatabaseLease")
805            .field(
806                "database_name",
807                &self.inner.as_ref().map(|inner| inner.name.as_str()),
808            )
809            .field("database_url", &"[REDACTED]")
810            .finish()
811    }
812}
813
814impl Drop for DatabaseLease {
815    fn drop(&mut self) {
816        if let Some(inner) = self.inner.take()
817            && let Err(error) = inner.queue_fallback_cleanup()
818        {
819            // Owned shutdown removes the containing server. On an external
820            // server the tagged residual is left for the next stale sweep.
821            log_cleanup(format_args!(
822                "fallback database cleanup could not be queued: {error}"
823            ));
824        }
825    }
826}
827
828struct DatabaseLeaseInner {
829    server: Arc<ServerInner>,
830    name: DatabaseName,
831    permit: OwnedSemaphorePermit,
832    prewarm_pool: Option<Weak<PrewarmPoolInner>>,
833}
834
835impl DatabaseLeaseInner {
836    fn queue_awaited_cleanup(
837        self,
838    ) -> Result<tokio::sync::oneshot::Receiver<crate::cleanup::AwaitedCleanupOutcome>> {
839        let Self {
840            server,
841            name,
842            permit,
843            prewarm_pool,
844        } = self;
845        let database_name = name.as_str().to_owned();
846        let database_cleanup = server.database_cleanup.clone();
847        let prewarm_return = prewarm_pool
848            .and_then(|pool| pool.upgrade())
849            .map(|pool| pool.begin_return());
850        database_cleanup.submit_awaited_outcome(
851            database_name,
852            permit,
853            database_cleanup_operation(server, name, prewarm_return),
854        )
855    }
856
857    fn queue_deferred_cleanup(self) -> Result<()> {
858        let Self {
859            server,
860            name,
861            permit,
862            prewarm_pool,
863        } = self;
864        let database_name = name.as_str().to_owned();
865        let database_cleanup = server.database_cleanup.clone();
866        let prewarm_return = prewarm_pool
867            .and_then(|pool| pool.upgrade())
868            .map(|pool| pool.begin_return());
869        database_cleanup.submit_deferred_outcome(
870            database_name,
871            Some(permit),
872            database_cleanup_operation(server, name, prewarm_return),
873        )
874    }
875
876    fn queue_fallback_cleanup(self) -> Result<()> {
877        let Self {
878            server,
879            name,
880            permit,
881            prewarm_pool,
882        } = self;
883        let database_name = name.as_str().to_owned();
884        let database_cleanup = server.database_cleanup.clone();
885        let prewarm_return = prewarm_pool
886            .and_then(|pool| pool.upgrade())
887            .map(|pool| pool.begin_return());
888        database_cleanup.submit_fallback_outcome(
889            database_name,
890            permit,
891            database_cleanup_operation(server, name, prewarm_return),
892        )
893    }
894}
895
896fn database_cleanup_operation(
897    server: Arc<ServerInner>,
898    name: DatabaseName,
899    mut prewarm_return: Option<PrewarmReturn>,
900) -> impl FnOnce() -> CleanupOutcome + Send + 'static {
901    move || {
902        let operation_server = server.clone();
903        let outcome = server.with_lifecycle_admin_disposition(
904            "connect for disposable database cleanup",
905            move |client| {
906                if let Err(error) = drop_database(client, &name) {
907                    return AdminSessionDisposition::Evict(CleanupOutcome::residual_possible(
908                        error,
909                    ));
910                }
911                let Some(ref mut refill) = prewarm_return else {
912                    return AdminSessionDisposition::Reuse(CleanupOutcome::succeeded());
913                };
914                if !refill.after_drop() {
915                    return AdminSessionDisposition::Reuse(CleanupOutcome::succeeded());
916                }
917
918                let fresh_name = DatabaseName::test(&operation_server.project);
919                if let Err(error) = operation_server.create_managed_database_classified(
920                    client,
921                    &fresh_name,
922                    refill.pool.template.name().as_str(),
923                    &ResourceMetadata::test(
924                        operation_server.project.clone(),
925                        operation_server.owner_key,
926                    ),
927                ) {
928                    return match error {
929                        ManagedDatabaseCreationFailure::NoResidual(error) => {
930                            AdminSessionDisposition::Reuse(CleanupOutcome::no_residual(error))
931                        }
932                        ManagedDatabaseCreationFailure::ResidualPossible(error) => {
933                            AdminSessionDisposition::Evict(CleanupOutcome::residual_possible(error))
934                        }
935                    };
936                }
937                let prepared = PreparedDatabase {
938                    database_url: operation_server.admin_url.database_url(&fresh_name),
939                    name: fresh_name,
940                };
941                if let Some((prepared, deletion)) = refill.publish(prepared) {
942                    // The pool closed after reserving this replacement. Remove it
943                    // on the same admin session so the enclosing cleanup barrier
944                    // still covers the complete slot transition.
945                    let result = drop_database(client, &prepared.name);
946                    deletion.finish();
947                    if let Err(error) = result {
948                        return AdminSessionDisposition::Evict(CleanupOutcome::residual_possible(
949                            error,
950                        ));
951                    }
952                }
953                AdminSessionDisposition::Reuse(CleanupOutcome::succeeded())
954            },
955        );
956        match outcome {
957            Ok(outcome) => outcome,
958            Err(error) => CleanupOutcome::residual_possible(error),
959        }
960    }
961}
962
963enum TemplatePreparation {
964    Ready(PersistentClient),
965    Initialize(TemplateInitialization),
966}
967
968struct TemplateInitialization {
969    server: Arc<ServerInner>,
970    name: DatabaseName,
971    fingerprint: TemplateFingerprint,
972    lock_key: i64,
973    coordination_key: i64,
974    exclusive_lock: PersistentClient,
975}
976
977impl TemplateInitialization {
978    fn finish(self) -> Result<PersistentClient> {
979        let Self {
980            server,
981            name,
982            fingerprint,
983            lock_key,
984            coordination_key,
985            mut exclusive_lock,
986        } = self;
987        let client = exclusive_lock.client_mut();
988        disable_database_connections(client, &name)?;
989        terminate_database_connections(client, &name, server.operation_timeout)?;
990        set_database_metadata(
991            client,
992            &name,
993            &ResourceMetadata::template(
994                server.project.clone(),
995                lock_key,
996                fingerprint,
997                TemplateState::Ready,
998            ),
999        )?;
1000        release_advisory_lock(client, lock_key)?;
1001        let ready = acquire_and_inspect_shared_template_lock(
1002            client,
1003            &server,
1004            &name,
1005            fingerprint,
1006            lock_key,
1007        )?;
1008        release_advisory_lock(client, coordination_key)?;
1009        if ready {
1010            Ok(exclusive_lock)
1011        } else {
1012            Err(Error::InconsistentMetadata {
1013                database_name: name.as_str().to_owned(),
1014            })
1015        }
1016    }
1017
1018    fn abort(self) -> Result<()> {
1019        let Self {
1020            name,
1021            lock_key,
1022            coordination_key,
1023            mut exclusive_lock,
1024            ..
1025        } = self;
1026        let client = exclusive_lock.client_mut();
1027        let cleanup = drop_database(client, &name);
1028        let unlock_template = release_advisory_lock(client, lock_key);
1029        let unlock_coordination = release_advisory_lock(client, coordination_key);
1030        cleanup.and(unlock_template).and(unlock_coordination)
1031    }
1032}
1033
1034fn begin_template(
1035    server: Arc<ServerInner>,
1036    name: DatabaseName,
1037    fingerprint: TemplateFingerprint,
1038) -> Result<TemplatePreparation> {
1039    let lock_key = template_lock_key(&server.project, fingerprint);
1040    let coordination_key = template_coordination_key(&server.project, fingerprint);
1041    let mut persistent_client = PersistentClient::new(connect_admin(
1042        &server.admin_url,
1043        server.operation_timeout,
1044        "connect for PostgreSQL template coordination",
1045    )?);
1046    loop {
1047        let client = persistent_client.client_mut();
1048        if acquire_and_inspect_shared_template_lock(client, &server, &name, fingerprint, lock_key)?
1049        {
1050            return Ok(TemplatePreparation::Ready(persistent_client));
1051        }
1052
1053        // Serialize upgrades so a retained ready shared lock cannot strand an
1054        // earlier coordination attempt waiting for the exclusive lock.
1055        acquire_template_advisory_lock(
1056            client,
1057            coordination_key,
1058            server.template_wait_timeout,
1059            server.operation_timeout,
1060        )?;
1061        if acquire_and_inspect_shared_template_lock(client, &server, &name, fingerprint, lock_key)?
1062        {
1063            release_advisory_lock(client, coordination_key)?;
1064            return Ok(TemplatePreparation::Ready(persistent_client));
1065        }
1066
1067        acquire_template_advisory_lock(
1068            client,
1069            lock_key,
1070            server.template_wait_timeout,
1071            server.operation_timeout,
1072        )?;
1073        if template_is_ready(client, &server, &name, fingerprint, lock_key)? {
1074            release_advisory_lock(client, lock_key)?;
1075            let ready = acquire_and_inspect_shared_template_lock(
1076                client,
1077                &server,
1078                &name,
1079                fingerprint,
1080                lock_key,
1081            )?;
1082            release_advisory_lock(client, coordination_key)?;
1083            if ready {
1084                return Ok(TemplatePreparation::Ready(persistent_client));
1085            }
1086            continue;
1087        }
1088        if let Some(record) = find_database(client, &name)? {
1089            let recoverable = record
1090                .comment
1091                .as_deref()
1092                .and_then(ResourceMetadata::parse)
1093                .is_some_and(|metadata| {
1094                    recoverable_template_initialization(
1095                        &metadata,
1096                        &server.project,
1097                        fingerprint,
1098                        lock_key,
1099                    )
1100                });
1101            if !recoverable {
1102                return Err(Error::InconsistentMetadata {
1103                    database_name: name.as_str().to_owned(),
1104                });
1105            }
1106            drop_database(client, &name)?;
1107        }
1108        server
1109            .create_managed_database_classified(
1110                client,
1111                &name,
1112                "template0",
1113                &ResourceMetadata::template(
1114                    server.project.clone(),
1115                    lock_key,
1116                    fingerprint,
1117                    TemplateState::Initializing,
1118                ),
1119            )
1120            .map_err(ManagedDatabaseCreationFailure::into_error)?;
1121        return Ok(TemplatePreparation::Initialize(TemplateInitialization {
1122            server,
1123            name,
1124            fingerprint,
1125            lock_key,
1126            coordination_key,
1127            exclusive_lock: persistent_client,
1128        }));
1129    }
1130}
1131
1132fn template_lock_key(project: &ProjectName, fingerprint: TemplateFingerprint) -> i64 {
1133    advisory_key(
1134        "template",
1135        &format!("{}:{}", project.as_str(), fingerprint.to_hex()),
1136    )
1137}
1138
1139fn template_coordination_key(project: &ProjectName, fingerprint: TemplateFingerprint) -> i64 {
1140    advisory_key(
1141        "template-coordination",
1142        &format!("{}:{}", project.as_str(), fingerprint.to_hex()),
1143    )
1144}
1145
1146fn acquire_and_inspect_shared_template_lock(
1147    client: &mut AdminClient,
1148    server: &ServerInner,
1149    name: &DatabaseName,
1150    fingerprint: TemplateFingerprint,
1151    lock_key: i64,
1152) -> Result<bool> {
1153    acquire_shared_template_advisory_lock(
1154        client,
1155        lock_key,
1156        server.template_wait_timeout,
1157        server.operation_timeout,
1158    )?;
1159    if template_is_ready(client, server, name, fingerprint, lock_key)? {
1160        Ok(true)
1161    } else {
1162        release_shared_advisory_lock(client, lock_key)?;
1163        Ok(false)
1164    }
1165}
1166
1167fn recoverable_template_initialization(
1168    metadata: &ResourceMetadata,
1169    project: &ProjectName,
1170    fingerprint: TemplateFingerprint,
1171    lock_key: i64,
1172) -> bool {
1173    matches!(
1174        metadata,
1175        ResourceMetadata::Template {
1176            project: metadata_project,
1177            lock_key: metadata_lock_key,
1178            fingerprint: metadata_fingerprint,
1179            state: TemplateState::Initializing,
1180            ..
1181        } if metadata_project == project
1182            && *metadata_lock_key == lock_key
1183            && *metadata_fingerprint == fingerprint
1184    )
1185}
1186
1187fn template_is_ready(
1188    client: &mut AdminClient,
1189    server: &ServerInner,
1190    name: &DatabaseName,
1191    fingerprint: TemplateFingerprint,
1192    lock_key: i64,
1193) -> Result<bool> {
1194    let record = find_database(client, name)?;
1195    Ok(template_record_is_ready(
1196        record.as_ref(),
1197        &server.project,
1198        fingerprint,
1199        lock_key,
1200    ))
1201}
1202
1203fn template_record_is_ready(
1204    record: Option<&DatabaseRecord>,
1205    project: &ProjectName,
1206    fingerprint: TemplateFingerprint,
1207    lock_key: i64,
1208) -> bool {
1209    let Some(comment) = record.and_then(|record| record.comment.as_deref()) else {
1210        return false;
1211    };
1212    matches!(
1213        ResourceMetadata::parse(comment),
1214        Some(ResourceMetadata::Template {
1215            project: metadata_project,
1216            lock_key: metadata_lock_key,
1217            fingerprint: metadata_fingerprint,
1218            state: TemplateState::Ready,
1219            ..
1220        }) if metadata_project == *project
1221            && metadata_lock_key == lock_key
1222            && metadata_fingerprint == fingerprint
1223    )
1224}
1225
1226async fn create_test_database(
1227    server: Arc<ServerInner>,
1228    template_name: &str,
1229) -> Result<DatabaseLease> {
1230    let permit = server.acquire_database_permit().await?;
1231    let template_name = template_name.to_owned();
1232    let name = DatabaseName::test(&server.project);
1233    let database_url = server.admin_url.database_url(&name);
1234    run_blocking(move || {
1235        server.with_lifecycle_admin("connect for disposable database creation", |client| {
1236            server
1237                .create_managed_database_classified(
1238                    client,
1239                    &name,
1240                    &template_name,
1241                    &ResourceMetadata::test(server.project.clone(), server.owner_key),
1242                )
1243                .map_err(ManagedDatabaseCreationFailure::into_error)
1244        })?;
1245        Ok(DatabaseLease {
1246            inner: Some(DatabaseLeaseInner {
1247                server,
1248                name,
1249                permit,
1250                prewarm_pool: None,
1251            }),
1252            database_url,
1253        })
1254    })
1255    .await
1256}
1257
1258async fn create_unpublished_database(
1259    server: Arc<ServerInner>,
1260    template_name: String,
1261) -> Result<UnpublishedDatabase> {
1262    run_blocking(move || {
1263        let name = DatabaseName::test(&server.project);
1264        let database_url = server.admin_url.database_url(&name);
1265        server.with_lifecycle_admin("connect for prewarmed database creation", |client| {
1266            server
1267                .create_managed_database_classified(
1268                    client,
1269                    &name,
1270                    &template_name,
1271                    &ResourceMetadata::test(server.project.clone(), server.owner_key),
1272                )
1273                .map_err(ManagedDatabaseCreationFailure::into_error)
1274        })?;
1275        Ok(UnpublishedDatabase {
1276            server,
1277            prepared: Some(PreparedDatabase { name, database_url }),
1278        })
1279    })
1280    .await
1281}
1282
1283fn queue_unpublished_database(server: Arc<ServerInner>, prepared: PreparedDatabase) {
1284    let database_name = prepared.name.as_str().to_owned();
1285    let database_cleanup = server.database_cleanup.clone();
1286    if let Err(error) = database_cleanup.submit_prewarmer(database_name, move || {
1287        server.with_lifecycle_admin("connect for unpublished prewarm cleanup", |client| {
1288            drop_database(client, &prepared.name)
1289        })
1290    }) {
1291        // Owned shutdown removes the containing server. On an external server,
1292        // the tagged residual remains eligible for owner-aware stale cleanup.
1293        log_cleanup(format_args!(
1294            "unpublished prewarmed database cleanup could not be queued: {error}"
1295        ));
1296    }
1297}
1298
1299fn queue_closed_prepared_database(pool: Arc<PrewarmPoolInner>, prepared: PreparedDatabase) {
1300    let server = pool.template.server().clone();
1301    let deletion = ClosedPrewarmDeletion { pool, active: true };
1302    queue_counted_prepared_database(server, prepared, deletion);
1303}
1304
1305fn queue_counted_prepared_database(
1306    server: Arc<ServerInner>,
1307    prepared: PreparedDatabase,
1308    deletion: ClosedPrewarmDeletion,
1309) {
1310    let database_name = prepared.name.as_str().to_owned();
1311    let database_cleanup = server.database_cleanup.clone();
1312    if let Err(error) = database_cleanup.submit_prewarmer(database_name, move || {
1313        let result = server.with_lifecycle_admin("connect for prewarm pool shutdown", |client| {
1314            drop_database(client, &prepared.name)
1315        });
1316        deletion.finish();
1317        result
1318    }) {
1319        log_cleanup(format_args!(
1320            "idle prewarmed database cleanup could not be queued: {error}"
1321        ));
1322    }
1323}
1324
1325/// Counts from an owner-aware stale database cleanup pass.
1326#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
1327#[non_exhaustive]
1328pub struct CleanupReport {
1329    pub dropped_test_databases: usize,
1330    pub dropped_templates: usize,
1331    pub skipped_active: usize,
1332    pub skipped_fresh: usize,
1333    pub skipped_unrecognized: usize,
1334}
1335
1336impl CleanupReport {
1337    pub fn total_dropped(self) -> usize {
1338        self.dropped_test_databases + self.dropped_templates
1339    }
1340}
1341
1342/// Drops only old, tagged resources whose PostgreSQL owner lock is not held.
1343pub async fn cleanup_stale_databases(
1344    admin_database_url: &str,
1345    project: impl Into<String>,
1346    stale_after: Duration,
1347) -> Result<CleanupReport> {
1348    let admin_url = AdminDatabaseUrl::parse(admin_database_url)?;
1349    let project = ProjectName::new(project)?;
1350    cleanup_stale_with(
1351        admin_url,
1352        project,
1353        stale_after,
1354        DEFAULT_CLEANUP_OPERATION_TIMEOUT,
1355    )
1356    .await
1357}
1358
1359async fn cleanup_stale_with(
1360    admin_url: AdminDatabaseUrl,
1361    project: ProjectName,
1362    stale_after: Duration,
1363    operation_timeout: Duration,
1364) -> Result<CleanupReport> {
1365    run_blocking(move || {
1366        cleanup_stale_blocking(&admin_url, &project, stale_after, operation_timeout)
1367    })
1368    .await
1369}
1370
1371fn cleanup_stale_blocking(
1372    admin_url: &AdminDatabaseUrl,
1373    project: &ProjectName,
1374    stale_after: Duration,
1375    operation_timeout: Duration,
1376) -> Result<CleanupReport> {
1377    let mut client = connect_admin(
1378        admin_url,
1379        operation_timeout,
1380        "connect for stale PostgreSQL database cleanup",
1381    )?;
1382    validate_postgres_18(&mut client)?;
1383    let records = list_databases(&mut client)?;
1384    let now = SystemTime::now()
1385        .duration_since(SystemTime::UNIX_EPOCH)
1386        .unwrap_or_default()
1387        .as_secs();
1388    let mut report = CleanupReport::default();
1389
1390    for record in records {
1391        let lock_key = match classify_cleanup_record(project, &record, now, stale_after) {
1392            CleanupClassification::Ignore => continue,
1393            CleanupClassification::Unrecognized => {
1394                report.skipped_unrecognized += 1;
1395                continue;
1396            }
1397            CleanupClassification::Fresh => {
1398                report.skipped_fresh += 1;
1399                continue;
1400            }
1401            CleanupClassification::Candidate { lock_key, .. } => lock_key,
1402        };
1403        if !try_acquire_advisory_lock(&mut client, lock_key)? {
1404            report.skipped_active += 1;
1405            continue;
1406        }
1407        let cleanup_result = cleanup_candidate_after_lock(
1408            &mut client,
1409            project,
1410            &record,
1411            lock_key,
1412            now,
1413            stale_after,
1414            &mut report,
1415        );
1416        let Some(dropped_kind) = finish_locked_cleanup(cleanup_result, || {
1417            release_advisory_lock(&mut client, lock_key)
1418        })?
1419        else {
1420            continue;
1421        };
1422        match dropped_kind {
1423            DatabaseKind::Test => report.dropped_test_databases += 1,
1424            DatabaseKind::Template => report.dropped_templates += 1,
1425        }
1426    }
1427    Ok(report)
1428}
1429
1430fn cleanup_candidate_after_lock(
1431    client: &mut AdminClient,
1432    project: &ProjectName,
1433    snapshot: &DatabaseRecord,
1434    held_key: i64,
1435    now: u64,
1436    stale_after: Duration,
1437    report: &mut CleanupReport,
1438) -> Result<Option<DatabaseKind>> {
1439    let snapshot_name = DatabaseName::from_existing(project, snapshot.name.clone())
1440        .expect("classified cleanup candidate has a valid managed database name");
1441    let current = find_database(client, &snapshot_name)?;
1442    let Some(name) = revalidate_cleanup_candidate(
1443        project,
1444        snapshot,
1445        current.as_ref(),
1446        held_key,
1447        now,
1448        stale_after,
1449        report,
1450    ) else {
1451        return Ok(None);
1452    };
1453    drop_database(client, &name)?;
1454    Ok(Some(name.kind()))
1455}
1456
1457fn finish_locked_cleanup<T>(
1458    operation: Result<T>,
1459    unlock: impl FnOnce() -> Result<()>,
1460) -> Result<T> {
1461    let unlock = unlock();
1462    match operation {
1463        Ok(value) => {
1464            unlock?;
1465            Ok(value)
1466        }
1467        Err(error) => Err(error),
1468    }
1469}
1470
1471enum CleanupClassification {
1472    Ignore,
1473    Unrecognized,
1474    Fresh,
1475    Candidate { name: DatabaseName, lock_key: i64 },
1476}
1477
1478fn classify_cleanup_record(
1479    project: &ProjectName,
1480    record: &DatabaseRecord,
1481    now: u64,
1482    stale_after: Duration,
1483) -> CleanupClassification {
1484    let Some(name) = DatabaseName::from_existing(project, record.name.clone()) else {
1485        return CleanupClassification::Ignore;
1486    };
1487    let Some(metadata) = record.comment.as_deref().and_then(ResourceMetadata::parse) else {
1488        return CleanupClassification::Unrecognized;
1489    };
1490    if metadata.project() != project || !metadata_matches_name(&metadata, &name) {
1491        return CleanupClassification::Unrecognized;
1492    }
1493    if now.saturating_sub(metadata.created_at()) < stale_after.as_secs() {
1494        return CleanupClassification::Fresh;
1495    }
1496    CleanupClassification::Candidate {
1497        name,
1498        lock_key: metadata.lock_key(),
1499    }
1500}
1501
1502fn revalidate_cleanup_candidate(
1503    project: &ProjectName,
1504    snapshot: &DatabaseRecord,
1505    current: Option<&DatabaseRecord>,
1506    held_key: i64,
1507    now: u64,
1508    stale_after: Duration,
1509    report: &mut CleanupReport,
1510) -> Option<DatabaseName> {
1511    let current = current?;
1512    match classify_cleanup_record(project, current, now, stale_after) {
1513        CleanupClassification::Ignore => None,
1514        CleanupClassification::Unrecognized => {
1515            report.skipped_unrecognized += 1;
1516            None
1517        }
1518        CleanupClassification::Fresh => {
1519            report.skipped_fresh += 1;
1520            None
1521        }
1522        CleanupClassification::Candidate { name, lock_key }
1523            if current == snapshot && lock_key == held_key =>
1524        {
1525            Some(name)
1526        }
1527        CleanupClassification::Candidate { .. } => None,
1528    }
1529}
1530
1531fn metadata_matches_name(metadata: &ResourceMetadata, name: &DatabaseName) -> bool {
1532    match (metadata, name.kind()) {
1533        (ResourceMetadata::Test { .. }, DatabaseKind::Test) => true,
1534        (
1535            ResourceMetadata::Template {
1536                project,
1537                fingerprint,
1538                ..
1539            },
1540            DatabaseKind::Template,
1541        ) => DatabaseName::template(project, *fingerprint) == *name,
1542        _ => false,
1543    }
1544}
1545
1546#[cfg(test)]
1547mod tests {
1548    use std::{
1549        sync::{
1550            Arc,
1551            atomic::{AtomicBool, Ordering},
1552            mpsc,
1553        },
1554        time::Duration,
1555    };
1556
1557    use tokio::sync::{Semaphore, oneshot};
1558
1559    use super::{
1560        CleanupClassification, CleanupReport, PreparedDatabase, PrewarmPoolState,
1561        PrewarmReturnPhase, acquire_database_permit_or_pool_close, classify_cleanup_record,
1562        finish_locked_cleanup, recoverable_template_initialization, revalidate_cleanup_candidate,
1563        run_blocking, template_coordination_key, template_lock_key, template_record_is_ready,
1564    };
1565    use crate::{
1566        Error, FingerprintBuilder, ProjectName,
1567        admin::DatabaseRecord,
1568        metadata::{ResourceMetadata, TemplateState},
1569        name::{DatabaseKind, DatabaseName},
1570    };
1571
1572    struct DropProbe(Arc<AtomicBool>);
1573
1574    impl Drop for DropProbe {
1575        fn drop(&mut self) {
1576            self.0.store(true, Ordering::SeqCst);
1577        }
1578    }
1579
1580    #[tokio::test]
1581    async fn prewarm_close_wakes_database_capacity_wait() {
1582        let database_admission = Arc::new(Semaphore::new(0));
1583        let pool_closed = Arc::new(Semaphore::new(0));
1584        let (started_sender, started_receiver) = oneshot::channel();
1585        let waiting_admission = database_admission.clone();
1586        let waiting_close = pool_closed.clone();
1587        let waiter = tokio::spawn(async move {
1588            acquire_database_permit_or_pool_close(
1589                async move {
1590                    let _ = started_sender.send(());
1591                    waiting_admission
1592                        .acquire_owned()
1593                        .await
1594                        .map_err(|_| Error::ConnectionBudgetClosed)
1595                },
1596                waiting_close,
1597            )
1598            .await
1599        });
1600        started_receiver
1601            .await
1602            .expect("database-capacity wait should start");
1603
1604        pool_closed.close();
1605        let error = tokio::time::timeout(Duration::from_secs(1), waiter)
1606            .await
1607            .expect("pool closure should wake the database-capacity wait")
1608            .expect("capacity waiter should not panic")
1609            .expect_err("a closed prewarm pool must reject the checkout");
1610        assert!(matches!(error, Error::PrewarmPoolClosed));
1611    }
1612
1613    #[test]
1614    fn failed_prewarm_phase_begins_cleanup_for_every_ready_database() {
1615        let project = ProjectName::new("prewarm_failure").unwrap();
1616        let mut state = PrewarmPoolState::new();
1617        state.ready.extend((0..2).map(|_| {
1618            let name = DatabaseName::test(&project);
1619            PreparedDatabase {
1620                database_url: format!("postgres://localhost/{}", name.as_str()),
1621                name,
1622            }
1623        }));
1624        state.creating = 1;
1625
1626        let ready = state
1627            .fail_and_begin_cleanup(PrewarmReturnPhase::Creating, 3)
1628            .expect("the first failure starts terminal cleanup");
1629
1630        assert_eq!(ready.len(), 2);
1631        assert!(state.ready.is_empty());
1632        assert_eq!(state.creating, 0);
1633        assert_eq!(state.deleting, 2);
1634        assert_eq!(state.occupied_slots(), 2);
1635        assert!(!state.accepting);
1636        assert!(state.cleanup_started);
1637        assert!(state.begin_cleanup(3).is_none());
1638    }
1639
1640    fn test_record(
1641        project: &ProjectName,
1642        name: &DatabaseName,
1643        owner_key: i64,
1644        created_at: u64,
1645    ) -> DatabaseRecord {
1646        DatabaseRecord {
1647            name: name.as_str().to_owned(),
1648            comment: Some(
1649                ResourceMetadata::Test {
1650                    project: project.clone(),
1651                    owner_key,
1652                    created_at,
1653                }
1654                .encode(),
1655            ),
1656        }
1657    }
1658
1659    fn template_record(
1660        project: &ProjectName,
1661        fingerprint: crate::TemplateFingerprint,
1662        lock_key: i64,
1663        state: TemplateState,
1664        created_at: u64,
1665    ) -> DatabaseRecord {
1666        DatabaseRecord {
1667            name: DatabaseName::template(project, fingerprint)
1668                .as_str()
1669                .to_owned(),
1670            comment: Some(
1671                ResourceMetadata::Template {
1672                    project: project.clone(),
1673                    lock_key,
1674                    fingerprint,
1675                    state,
1676                    created_at,
1677                }
1678                .encode(),
1679            ),
1680        }
1681    }
1682
1683    #[tokio::test(flavor = "multi_thread")]
1684    async fn detached_blocking_results_are_dropped() {
1685        let cleaned = Arc::new(AtomicBool::new(false));
1686        let cleaned_for_task = cleaned.clone();
1687        let (started_sender, started_receiver) = oneshot::channel();
1688        let (release_sender, release_receiver) = mpsc::channel();
1689
1690        let task = tokio::spawn(async move {
1691            run_blocking(move || {
1692                let _ = started_sender.send(());
1693                release_receiver.recv().unwrap();
1694                Ok(DropProbe(cleaned_for_task))
1695            })
1696            .await
1697        });
1698
1699        started_receiver.await.unwrap();
1700        task.abort();
1701        release_sender.send(()).unwrap();
1702        let join_error = match task.await {
1703            Ok(_) => panic!("cancelled task unexpectedly completed"),
1704            Err(error) => error,
1705        };
1706        assert!(join_error.is_cancelled());
1707        tokio::time::timeout(Duration::from_secs(1), async {
1708            while !cleaned.load(Ordering::SeqCst) {
1709                tokio::task::yield_now().await;
1710            }
1711        })
1712        .await
1713        .expect("detached blocking result should be cleaned");
1714    }
1715
1716    #[test]
1717    fn template_lock_domains_are_distinct_and_namespaced_by_project() {
1718        let fingerprint = FingerprintBuilder::new("schema").finish();
1719        let left = template_lock_key(&ProjectName::new("left").unwrap(), fingerprint);
1720        let right = template_lock_key(&ProjectName::new("right").unwrap(), fingerprint);
1721        let coordination =
1722            template_coordination_key(&ProjectName::new("left").unwrap(), fingerprint);
1723        assert_ne!(left, right);
1724        assert_ne!(left, coordination);
1725    }
1726
1727    #[test]
1728    fn only_matching_initializing_templates_are_recoverable() {
1729        let project = ProjectName::new("creditkit").unwrap();
1730        let fingerprint = FingerprintBuilder::new("schema").finish();
1731        let lock_key = template_lock_key(&project, fingerprint);
1732        let initializing = ResourceMetadata::template(
1733            project.clone(),
1734            lock_key,
1735            fingerprint,
1736            TemplateState::Initializing,
1737        );
1738        let ready = ResourceMetadata::template(
1739            project.clone(),
1740            lock_key,
1741            fingerprint,
1742            TemplateState::Ready,
1743        );
1744        assert!(recoverable_template_initialization(
1745            &initializing,
1746            &project,
1747            fingerprint,
1748            lock_key
1749        ));
1750        assert!(!recoverable_template_initialization(
1751            &ready,
1752            &project,
1753            fingerprint,
1754            lock_key
1755        ));
1756        assert!(!recoverable_template_initialization(
1757            &initializing,
1758            &ProjectName::new("another").unwrap(),
1759            fingerprint,
1760            lock_key
1761        ));
1762    }
1763
1764    #[test]
1765    fn exact_template_records_preserve_catalog_states() {
1766        let project = ProjectName::new("creditkit").unwrap();
1767        let fingerprint = FingerprintBuilder::new("schema").finish();
1768        let lock_key = template_lock_key(&project, fingerprint);
1769        let name = DatabaseName::template(&project, fingerprint)
1770            .as_str()
1771            .to_owned();
1772        let untagged = DatabaseRecord {
1773            name: name.clone(),
1774            comment: None,
1775        };
1776        let initializing = DatabaseRecord {
1777            name: name.clone(),
1778            comment: Some(
1779                ResourceMetadata::template(
1780                    project.clone(),
1781                    lock_key,
1782                    fingerprint,
1783                    TemplateState::Initializing,
1784                )
1785                .encode(),
1786            ),
1787        };
1788        let ready = DatabaseRecord {
1789            name,
1790            comment: Some(
1791                ResourceMetadata::template(
1792                    project.clone(),
1793                    lock_key,
1794                    fingerprint,
1795                    TemplateState::Ready,
1796                )
1797                .encode(),
1798            ),
1799        };
1800
1801        assert!(!template_record_is_ready(
1802            None,
1803            &project,
1804            fingerprint,
1805            lock_key
1806        ));
1807        assert!(!template_record_is_ready(
1808            Some(&untagged),
1809            &project,
1810            fingerprint,
1811            lock_key
1812        ));
1813        assert!(!template_record_is_ready(
1814            Some(&initializing),
1815            &project,
1816            fingerprint,
1817            lock_key
1818        ));
1819        assert!(template_record_is_ready(
1820            Some(&ready),
1821            &project,
1822            fingerprint,
1823            lock_key
1824        ));
1825    }
1826
1827    #[test]
1828    fn cleanup_classification_is_conservative_without_postgres() {
1829        let project = ProjectName::new("creditkit").unwrap();
1830        let test_name = DatabaseName::test(&project);
1831        let metadata = ResourceMetadata::test(project.clone(), 42);
1832        let created_at = metadata.created_at();
1833
1834        let unrecognized = classify_cleanup_record(
1835            &project,
1836            &DatabaseRecord {
1837                name: test_name.as_str().to_owned(),
1838                comment: None,
1839            },
1840            created_at + 100,
1841            Duration::from_secs(10),
1842        );
1843        assert!(matches!(unrecognized, CleanupClassification::Unrecognized));
1844
1845        let fresh = classify_cleanup_record(
1846            &project,
1847            &DatabaseRecord {
1848                name: test_name.as_str().to_owned(),
1849                comment: Some(metadata.encode()),
1850            },
1851            created_at + 5,
1852            Duration::from_secs(10),
1853        );
1854        assert!(matches!(fresh, CleanupClassification::Fresh));
1855
1856        let candidate = classify_cleanup_record(
1857            &project,
1858            &DatabaseRecord {
1859                name: test_name.as_str().to_owned(),
1860                comment: Some(metadata.encode()),
1861            },
1862            created_at + 10,
1863            Duration::from_secs(10),
1864        );
1865        assert!(matches!(
1866            candidate,
1867            CleanupClassification::Candidate { lock_key: 42, .. }
1868        ));
1869
1870        let malformed_name = classify_cleanup_record(
1871            &project,
1872            &DatabaseRecord {
1873                name: "pgh_creditkit_test_backup".to_owned(),
1874                comment: Some(metadata.encode()),
1875            },
1876            created_at + 10,
1877            Duration::from_secs(10),
1878        );
1879        assert!(matches!(malformed_name, CleanupClassification::Ignore));
1880    }
1881
1882    #[test]
1883    fn locked_cleanup_requires_an_unchanged_candidate() {
1884        let project = ProjectName::new("creditkit").unwrap();
1885        let test_name = DatabaseName::test(&project);
1886        let snapshot = test_record(&project, &test_name, 42, 10);
1887        let mut report = CleanupReport::default();
1888
1889        let unchanged = revalidate_cleanup_candidate(
1890            &project,
1891            &snapshot,
1892            Some(&snapshot),
1893            42,
1894            100,
1895            Duration::from_secs(10),
1896            &mut report,
1897        );
1898        assert_eq!(
1899            unchanged.as_ref().map(DatabaseName::kind),
1900            Some(DatabaseKind::Test)
1901        );
1902
1903        let refreshed = test_record(&project, &test_name, 42, 95);
1904        assert!(
1905            revalidate_cleanup_candidate(
1906                &project,
1907                &snapshot,
1908                Some(&refreshed),
1909                42,
1910                100,
1911                Duration::from_secs(10),
1912                &mut report,
1913            )
1914            .is_none()
1915        );
1916        assert_eq!(report.skipped_fresh, 1);
1917
1918        let changed_key = test_record(&project, &test_name, 84, 10);
1919        assert!(
1920            revalidate_cleanup_candidate(
1921                &project,
1922                &snapshot,
1923                Some(&changed_key),
1924                42,
1925                100,
1926                Duration::from_secs(10),
1927                &mut report,
1928            )
1929            .is_none()
1930        );
1931        assert!(
1932            revalidate_cleanup_candidate(
1933                &project,
1934                &snapshot,
1935                None,
1936                42,
1937                100,
1938                Duration::from_secs(10),
1939                &mut report,
1940            )
1941            .is_none()
1942        );
1943    }
1944
1945    #[test]
1946    fn locked_cleanup_rejects_a_finalized_template_snapshot() {
1947        let project = ProjectName::new("creditkit").unwrap();
1948        let fingerprint = FingerprintBuilder::new("cleanup-race").finish();
1949        let initializing =
1950            template_record(&project, fingerprint, 42, TemplateState::Initializing, 10);
1951        let ready = template_record(&project, fingerprint, 42, TemplateState::Ready, 10);
1952        let mut report = CleanupReport::default();
1953
1954        assert!(
1955            revalidate_cleanup_candidate(
1956                &project,
1957                &initializing,
1958                Some(&ready),
1959                42,
1960                100,
1961                Duration::from_secs(10),
1962                &mut report,
1963            )
1964            .is_none()
1965        );
1966        assert_eq!(report, CleanupReport::default());
1967    }
1968
1969    #[test]
1970    fn locked_cleanup_keeps_the_operation_error_primary() {
1971        let unlocked = AtomicBool::new(false);
1972        let error = finish_locked_cleanup::<()>(
1973            Err(Error::InvalidConfiguration {
1974                reason: "locked reread",
1975            }),
1976            || {
1977                unlocked.store(true, Ordering::SeqCst);
1978                Err(Error::InvalidConfiguration { reason: "unlock" })
1979            },
1980        )
1981        .unwrap_err();
1982        assert!(unlocked.load(Ordering::SeqCst));
1983        assert!(matches!(
1984            error,
1985            Error::InvalidConfiguration {
1986                reason: "locked reread"
1987            }
1988        ));
1989
1990        let error = finish_locked_cleanup(Ok(()), || {
1991            Err(Error::InvalidConfiguration { reason: "unlock" })
1992        })
1993        .unwrap_err();
1994        assert!(matches!(
1995            error,
1996            Error::InvalidConfiguration { reason: "unlock" }
1997        ));
1998    }
1999}