1use std::{
2 collections::VecDeque,
3 future::Future,
4 sync::{
5 Arc, Mutex, Weak,
6 atomic::{AtomicUsize, Ordering},
7 },
8 task::Poll,
9 time::{Duration, SystemTime},
10};
11
12use tokio::sync::{OwnedSemaphorePermit, Semaphore};
13
14use crate::{
15 BoxError, ConnectionLimits, Error, HarnessConfig, ProjectName, Result, TemplateFingerprint,
16 TemplateSpec,
17 admin::{
18 AdminClient, AdminDatabaseUrl, AdminSessionDisposition, DatabaseRecord, PersistentClient,
19 acquire_shared_template_advisory_lock, acquire_template_advisory_lock, advisory_key,
20 connect_admin, disable_database_connections, drop_database, find_database, list_databases,
21 release_advisory_lock, release_shared_advisory_lock, set_database_metadata,
22 terminate_database_connections, try_acquire_advisory_lock, validate_postgres_18,
23 },
24 admission::ManagedDatabaseCreationFailure,
25 cleanup::{CleanupOutcome, log_cleanup},
26 metadata::{ResourceMetadata, TemplateState},
27 name::{DatabaseKind, DatabaseName},
28 server::{ServerInner, TemplateCacheAction, TemplateInner, run_blocking},
29};
30
31const DEFAULT_CLEANUP_OPERATION_TIMEOUT: Duration = Duration::from_secs(90);
32
33#[derive(Clone)]
35pub struct PostgresHarness {
36 server: Arc<ServerInner>,
37}
38
39impl PostgresHarness {
40 pub async fn start(config: HarnessConfig) -> Result<Self> {
41 let server = ServerInner::start(config).await?;
42 let harness = Self { server };
43 if harness.server.is_external() && harness.server.cleanup_on_start {
44 cleanup_stale_with(
45 harness.server.admin_url.clone(),
46 harness.server.project.clone(),
47 harness.server.stale_after,
48 harness.server.operation_timeout,
49 )
50 .await?;
51 }
52 Ok(harness)
53 }
54
55 pub fn project(&self) -> &ProjectName {
56 &self.server.project
57 }
58
59 pub fn admin_database_url(&self) -> &str {
60 self.server.admin_url.as_str()
61 }
62
63 pub fn is_external(&self) -> bool {
64 self.server.is_external()
65 }
66
67 pub fn container_id(&self) -> Option<&str> {
71 self.server.container_id()
72 }
73
74 pub fn connection_limits(&self) -> ConnectionLimits {
77 self.server.connection_limits
78 }
79
80 pub async fn shutdown(&self) -> Result<()> {
95 self.server.shutdown_container().await
96 }
97
98 pub async fn drain_deferred_cleanup(&self) -> Result<()> {
106 let database_cleanup = self.server.database_cleanup.clone();
107 let outcome = run_blocking(move || Ok(database_cleanup.begin_drain())).await?;
108 outcome.finish()
109 }
110
111 pub async fn empty_database(&self) -> Result<DatabaseLease> {
113 create_test_database(self.server.clone(), "template0").await
114 }
115
116 pub async fn template<F, Fut>(
124 &self,
125 spec: TemplateSpec,
126 initializer: F,
127 ) -> Result<DatabaseTemplate>
128 where
129 F: FnOnce(String) -> Fut,
130 Fut: Future<Output = std::result::Result<(), BoxError>>,
131 {
132 let fingerprint = spec.fingerprint();
133 let mut initializer = Some(initializer);
134 loop {
135 match self.server.template_cache_action(fingerprint) {
136 TemplateCacheAction::Ready(inner) => return Ok(DatabaseTemplate { inner }),
137 TemplateCacheAction::Wait(waiter) => {
138 if let Some(inner) = waiter.wait().await {
139 return Ok(DatabaseTemplate { inner });
140 }
141 }
142 TemplateCacheAction::Initialize(cache_initializer) => {
143 let name = DatabaseName::template(&self.server.project, fingerprint);
144 let server = self.server.clone();
145 let preparation =
146 run_blocking(move || begin_template(server, name, fingerprint)).await?;
147
148 let template_lock = match preparation {
149 TemplatePreparation::Ready(template_lock) => template_lock,
150 TemplatePreparation::Initialize(initialization) => {
151 let database_url = initialization
152 .server
153 .admin_url
154 .database_url(&initialization.name);
155 let initialize = initializer
156 .take()
157 .expect("a template caller initializes at most once");
158 match initialize(database_url).await {
159 Ok(()) => run_blocking(move || initialization.finish()).await?,
160 Err(initializer) => {
161 let cleanup =
162 run_blocking(move || initialization.abort()).await;
163 return match cleanup {
164 Ok(()) => Err(Error::TemplateInitializer {
165 source: initializer,
166 }),
167 Err(cleanup) => Err(Error::TemplateInitializerAndCleanup {
168 initializer,
169 cleanup: Box::new(cleanup),
170 }),
171 };
172 }
173 }
174 }
175 };
176
177 let inner = Arc::new(TemplateInner::new(
178 self.server.clone(),
179 DatabaseName::template(&self.server.project, fingerprint),
180 fingerprint,
181 template_lock,
182 ));
183 return Ok(DatabaseTemplate {
184 inner: cache_initializer.publish(inner),
185 });
186 }
187 }
188 }
189 }
190}
191
192#[derive(Clone)]
194pub struct DatabaseTemplate {
195 inner: Arc<TemplateInner>,
196}
197
198impl DatabaseTemplate {
199 pub fn database_name(&self) -> &str {
200 self.inner.name().as_str()
201 }
202
203 pub fn fingerprint(&self) -> TemplateFingerprint {
204 self.inner.fingerprint()
205 }
206
207 pub async fn database(&self) -> Result<DatabaseLease> {
208 create_test_database(self.inner.server().clone(), self.inner.name().as_str()).await
209 }
210
211 pub async fn prewarm(&self, capacity: usize) -> Result<PrewarmedDatabasePool> {
219 let max_capacity = self
220 .inner
221 .server()
222 .connection_limits
223 .max_simultaneous_leases();
224 if capacity == 0 || capacity > max_capacity {
225 return Err(Error::InvalidPrewarmCapacity {
226 capacity,
227 max_capacity,
228 });
229 }
230
231 let pool = PrewarmedDatabasePool::new(self.inner.clone(), capacity)?;
232 for _ in 0..capacity {
233 let creation = pool.inner.begin_creation()?;
234 let prepared = create_unpublished_database(
235 self.inner.server().clone(),
236 self.inner.name().as_str().to_owned(),
237 )
238 .await;
239 match prepared {
240 Ok(prepared) => creation.publish(prepared),
241 Err(error) => {
242 creation.fail();
243 return Err(error);
244 }
245 }
246 }
247 Ok(pool)
248 }
249}
250
251#[derive(Clone, Copy, Debug, Eq, PartialEq)]
253#[non_exhaustive]
254pub struct PrewarmPoolStatus {
255 capacity: usize,
256 ready: usize,
257 leased: usize,
258 creating: usize,
259 deleting: usize,
260 closed: bool,
261}
262
263impl PrewarmPoolStatus {
264 pub const fn capacity(&self) -> usize {
265 self.capacity
266 }
267
268 pub const fn ready(&self) -> usize {
269 self.ready
270 }
271
272 pub const fn leased(&self) -> usize {
273 self.leased
274 }
275
276 pub const fn creating(&self) -> usize {
277 self.creating
278 }
279
280 pub const fn deleting(&self) -> usize {
281 self.deleting
282 }
283
284 pub const fn is_closed(&self) -> bool {
285 self.closed
286 }
287
288 pub const fn occupied_slots(&self) -> usize {
291 self.ready + self.leased + self.creating + self.deleting
292 }
293}
294
295pub struct PrewarmedDatabasePool {
297 inner: Arc<PrewarmPoolInner>,
298}
299
300impl PrewarmedDatabasePool {
301 fn new(template: Arc<TemplateInner>, capacity: usize) -> Result<Self> {
302 let inner = Arc::new(PrewarmPoolInner {
303 template,
304 capacity,
305 public_handles: AtomicUsize::new(1),
306 ready_admission: Arc::new(Semaphore::new(0)),
307 closed: Arc::new(Semaphore::new(0)),
308 state: Mutex::new(PrewarmPoolState::new()),
309 });
310 inner.template.server().register_prewarm_pool(&inner)?;
311 Ok(Self { inner })
312 }
313
314 pub fn capacity(&self) -> usize {
316 self.inner.capacity
317 }
318
319 pub fn status(&self) -> PrewarmPoolStatus {
321 self.inner.status()
322 }
323
324 pub async fn database(&self) -> Result<DatabaseLease> {
331 let ready = self
332 .inner
333 .ready_admission
334 .clone()
335 .acquire_owned()
336 .await
337 .map_err(|_| Error::PrewarmPoolClosed)?;
338 let permit = acquire_database_permit_or_pool_close(
339 self.inner.template.server().acquire_database_permit(),
340 self.inner.closed.clone(),
341 )
342 .await?;
343 let Some(prepared) = self.inner.take_ready() else {
344 return Err(Error::PrewarmPoolClosed);
345 };
346 ready.forget();
347 Ok(prepared.into_lease(permit, Arc::downgrade(&self.inner)))
348 }
349
350 pub async fn shutdown(self) -> Result<()> {
356 self.inner.close_and_queue_ready();
357 let database_cleanup = self.inner.template.server().database_cleanup.clone();
358 let outcome = run_blocking(move || Ok(database_cleanup.begin_drain())).await?;
359 outcome.finish()
360 }
361}
362
363impl Clone for PrewarmedDatabasePool {
364 fn clone(&self) -> Self {
365 self.inner.public_handles.fetch_add(1, Ordering::Relaxed);
366 Self {
367 inner: self.inner.clone(),
368 }
369 }
370}
371
372impl std::fmt::Debug for PrewarmedDatabasePool {
373 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
374 formatter
375 .debug_struct("PrewarmedDatabasePool")
376 .field("template", &self.inner.template.name().as_str())
377 .field("status", &self.status())
378 .finish()
379 }
380}
381
382impl Drop for PrewarmedDatabasePool {
383 fn drop(&mut self) {
384 if self.inner.public_handles.fetch_sub(1, Ordering::AcqRel) == 1 {
385 self.inner.close_and_queue_ready();
386 }
387 }
388}
389
390pub(crate) struct PrewarmPoolInner {
391 template: Arc<TemplateInner>,
392 capacity: usize,
393 public_handles: AtomicUsize,
394 ready_admission: Arc<Semaphore>,
395 closed: Arc<Semaphore>,
396 state: Mutex<PrewarmPoolState>,
397}
398
399struct PrewarmPoolState {
400 ready: VecDeque<PreparedDatabase>,
401 leased: usize,
402 creating: usize,
403 deleting: usize,
404 accepting: bool,
405 cleanup_started: bool,
406}
407
408impl PrewarmPoolState {
409 fn new() -> Self {
410 Self {
411 ready: VecDeque::new(),
412 leased: 0,
413 creating: 0,
414 deleting: 0,
415 accepting: true,
416 cleanup_started: false,
417 }
418 }
419
420 fn occupied_slots(&self) -> usize {
421 self.ready.len() + self.leased + self.creating + self.deleting
422 }
423
424 fn fail_and_begin_cleanup(
425 &mut self,
426 phase: PrewarmReturnPhase,
427 capacity: usize,
428 ) -> Option<VecDeque<PreparedDatabase>> {
429 match phase {
430 PrewarmReturnPhase::Deleting => {
431 debug_assert!(self.deleting > 0);
432 self.deleting = self.deleting.saturating_sub(1);
433 }
434 PrewarmReturnPhase::Creating => {
435 debug_assert!(self.creating > 0);
436 self.creating = self.creating.saturating_sub(1);
437 }
438 PrewarmReturnPhase::Complete => return None,
439 }
440 self.begin_cleanup(capacity)
441 }
442
443 fn begin_cleanup(&mut self, capacity: usize) -> Option<VecDeque<PreparedDatabase>> {
444 self.accepting = false;
445 if self.cleanup_started {
446 return None;
447 }
448 self.cleanup_started = true;
449 let ready = std::mem::take(&mut self.ready);
450 self.deleting += ready.len();
451 debug_assert!(self.occupied_slots() <= capacity);
452 Some(ready)
453 }
454}
455
456impl PrewarmPoolInner {
457 fn lock_state(&self) -> std::sync::MutexGuard<'_, PrewarmPoolState> {
458 self.state
459 .lock()
460 .unwrap_or_else(std::sync::PoisonError::into_inner)
461 }
462
463 fn status(&self) -> PrewarmPoolStatus {
464 let state = self.lock_state();
465 PrewarmPoolStatus {
466 capacity: self.capacity,
467 ready: state.ready.len(),
468 leased: state.leased,
469 creating: state.creating,
470 deleting: state.deleting,
471 closed: !state.accepting,
472 }
473 }
474
475 fn begin_creation(self: &Arc<Self>) -> Result<PrewarmCreation> {
476 let mut state = self.lock_state();
477 if !state.accepting {
478 return Err(Error::PrewarmPoolClosed);
479 }
480 debug_assert!(state.occupied_slots() < self.capacity);
481 if state.occupied_slots() >= self.capacity {
482 return Err(Error::PrewarmPoolClosed);
483 }
484 state.creating += 1;
485 drop(state);
486 Ok(PrewarmCreation {
487 pool: self.clone(),
488 active: true,
489 })
490 }
491
492 fn take_ready(&self) -> Option<PreparedDatabase> {
493 let mut state = self.lock_state();
494 if !state.accepting {
495 return None;
496 }
497 let prepared = state.ready.pop_front()?;
498 state.leased += 1;
499 debug_assert!(state.occupied_slots() <= self.capacity);
500 Some(prepared)
501 }
502
503 fn begin_return(self: &Arc<Self>) -> PrewarmReturn {
504 let mut state = self.lock_state();
505 debug_assert!(state.leased > 0);
506 state.leased = state.leased.saturating_sub(1);
507 state.deleting += 1;
508 debug_assert!(state.occupied_slots() <= self.capacity);
509 drop(state);
510 PrewarmReturn {
511 pool: self.clone(),
512 phase: PrewarmReturnPhase::Deleting,
513 }
514 }
515
516 fn finish_creation(
517 self: &Arc<Self>,
518 prepared: PreparedDatabase,
519 ) -> Option<(PreparedDatabase, ClosedPrewarmDeletion)> {
520 let mut state = self.lock_state();
521 debug_assert!(state.creating > 0);
522 state.creating = state.creating.saturating_sub(1);
523 if state.accepting && !state.cleanup_started {
524 state.ready.push_back(prepared);
525 debug_assert!(state.occupied_slots() <= self.capacity);
526 drop(state);
527 self.ready_admission.add_permits(1);
528 None
529 } else {
530 state.deleting += 1;
531 debug_assert!(state.occupied_slots() <= self.capacity);
532 Some((
533 prepared,
534 ClosedPrewarmDeletion {
535 pool: self.clone(),
536 active: true,
537 },
538 ))
539 }
540 }
541
542 fn fail_phase(self: &Arc<Self>, phase: PrewarmReturnPhase) {
543 let ready = self
544 .lock_state()
545 .fail_and_begin_cleanup(phase, self.capacity);
546 self.finish_close(ready);
547 }
548
549 pub(crate) fn close_and_queue_ready(self: &Arc<Self>) {
550 let ready = self.lock_state().begin_cleanup(self.capacity);
551 self.finish_close(ready);
552 }
553
554 fn finish_close(self: &Arc<Self>, ready: Option<VecDeque<PreparedDatabase>>) {
555 self.ready_admission.close();
556 self.closed.close();
557 let Some(ready) = ready else {
558 return;
559 };
560 for prepared in ready {
561 queue_closed_prepared_database(self.clone(), prepared);
562 }
563 }
564}
565
566async fn acquire_database_permit_or_pool_close<F>(
567 database_admission: F,
568 pool_closed: Arc<Semaphore>,
569) -> Result<OwnedSemaphorePermit>
570where
571 F: Future<Output = Result<OwnedSemaphorePermit>>,
572{
573 let mut database_admission = std::pin::pin!(database_admission);
574 let mut pool_closed = std::pin::pin!(pool_closed.acquire_owned());
575 std::future::poll_fn(|context| {
576 if let Poll::Ready(result) = database_admission.as_mut().poll(context) {
577 return Poll::Ready(result);
578 }
579 if pool_closed.as_mut().poll(context).is_ready() {
580 return Poll::Ready(Err(Error::PrewarmPoolClosed));
581 }
582 Poll::Pending
583 })
584 .await
585}
586
587struct PreparedDatabase {
588 name: DatabaseName,
589 database_url: String,
590}
591
592impl PreparedDatabase {
593 fn into_lease(
594 self,
595 permit: OwnedSemaphorePermit,
596 prewarm_pool: Weak<PrewarmPoolInner>,
597 ) -> DatabaseLease {
598 DatabaseLease {
599 inner: Some(DatabaseLeaseInner {
600 server: prewarm_pool
601 .upgrade()
602 .expect("a leased prewarmed database retains its public pool")
603 .template
604 .server()
605 .clone(),
606 name: self.name,
607 permit,
608 prewarm_pool: Some(prewarm_pool),
609 }),
610 database_url: self.database_url,
611 }
612 }
613}
614
615struct UnpublishedDatabase {
616 server: Arc<ServerInner>,
617 prepared: Option<PreparedDatabase>,
618}
619
620impl UnpublishedDatabase {
621 fn publish(mut self) -> PreparedDatabase {
622 self.prepared
623 .take()
624 .expect("an unpublished database is published at most once")
625 }
626}
627
628impl Drop for UnpublishedDatabase {
629 fn drop(&mut self) {
630 if let Some(prepared) = self.prepared.take() {
631 queue_unpublished_database(self.server.clone(), prepared);
632 }
633 }
634}
635
636struct PrewarmCreation {
637 pool: Arc<PrewarmPoolInner>,
638 active: bool,
639}
640
641impl PrewarmCreation {
642 fn publish(mut self, unpublished: UnpublishedDatabase) {
643 let prepared = unpublished.publish();
644 if let Some((prepared, deletion)) = self.pool.finish_creation(prepared) {
645 queue_counted_prepared_database(
646 self.pool.template.server().clone(),
647 prepared,
648 deletion,
649 );
650 }
651 self.active = false;
652 }
653
654 fn fail(mut self) {
655 self.pool.fail_phase(PrewarmReturnPhase::Creating);
656 self.active = false;
657 }
658}
659
660impl Drop for PrewarmCreation {
661 fn drop(&mut self) {
662 if self.active {
663 self.pool.fail_phase(PrewarmReturnPhase::Creating);
664 }
665 }
666}
667
668#[derive(Clone, Copy, Debug, Eq, PartialEq)]
669enum PrewarmReturnPhase {
670 Deleting,
671 Creating,
672 Complete,
673}
674
675struct PrewarmReturn {
676 pool: Arc<PrewarmPoolInner>,
677 phase: PrewarmReturnPhase,
678}
679
680impl PrewarmReturn {
681 fn after_drop(&mut self) -> bool {
682 let mut state = self.pool.lock_state();
683 debug_assert_eq!(self.phase, PrewarmReturnPhase::Deleting);
684 debug_assert!(state.deleting > 0);
685 state.deleting = state.deleting.saturating_sub(1);
686 if state.accepting && !state.cleanup_started {
687 state.creating += 1;
688 self.phase = PrewarmReturnPhase::Creating;
689 debug_assert!(state.occupied_slots() <= self.pool.capacity);
690 true
691 } else {
692 self.phase = PrewarmReturnPhase::Complete;
693 false
694 }
695 }
696
697 fn publish(
698 &mut self,
699 prepared: PreparedDatabase,
700 ) -> Option<(PreparedDatabase, ClosedPrewarmDeletion)> {
701 debug_assert_eq!(self.phase, PrewarmReturnPhase::Creating);
702 let unpublished = self.pool.finish_creation(prepared);
703 self.phase = PrewarmReturnPhase::Complete;
704 unpublished
705 }
706}
707
708impl Drop for PrewarmReturn {
709 fn drop(&mut self) {
710 if self.phase != PrewarmReturnPhase::Complete {
711 self.pool.fail_phase(self.phase);
712 self.phase = PrewarmReturnPhase::Complete;
713 }
714 }
715}
716
717struct ClosedPrewarmDeletion {
718 pool: Arc<PrewarmPoolInner>,
719 active: bool,
720}
721
722impl ClosedPrewarmDeletion {
723 fn finish(mut self) {
724 let mut state = self.pool.lock_state();
725 debug_assert!(state.deleting > 0);
726 state.deleting = state.deleting.saturating_sub(1);
727 self.active = false;
728 }
729}
730
731impl Drop for ClosedPrewarmDeletion {
732 fn drop(&mut self) {
733 if self.active {
734 let mut state = self.pool.lock_state();
735 debug_assert!(state.deleting > 0);
736 state.deleting = state.deleting.saturating_sub(1);
737 }
738 }
739}
740
741pub struct DatabaseLease {
747 inner: Option<DatabaseLeaseInner>,
748 database_url: String,
749}
750
751impl DatabaseLease {
752 pub fn database_name(&self) -> &str {
753 self.inner
754 .as_ref()
755 .expect("database lease is present until cleanup consumes it")
756 .name
757 .as_str()
758 }
759
760 pub fn database_url(&self) -> &str {
761 &self.database_url
762 }
763
764 pub async fn cleanup(mut self) -> Result<()> {
771 let inner = self
772 .inner
773 .take()
774 .expect("database lease cleanup runs at most once");
775 let completion = run_blocking(move || inner.queue_awaited_cleanup()).await?;
776 completion
777 .await
778 .map_err(|_| Error::CleanupWorkerStopped)?
779 .finish()
780 }
781
782 pub async fn defer_cleanup(mut self) -> Result<()> {
793 let inner = self
794 .inner
795 .take()
796 .expect("database lease cleanup runs at most once");
797 run_blocking(move || inner.queue_deferred_cleanup()).await
798 }
799}
800
801impl std::fmt::Debug for DatabaseLease {
802 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
803 formatter
804 .debug_struct("DatabaseLease")
805 .field(
806 "database_name",
807 &self.inner.as_ref().map(|inner| inner.name.as_str()),
808 )
809 .field("database_url", &"[REDACTED]")
810 .finish()
811 }
812}
813
814impl Drop for DatabaseLease {
815 fn drop(&mut self) {
816 if let Some(inner) = self.inner.take()
817 && let Err(error) = inner.queue_fallback_cleanup()
818 {
819 log_cleanup(format_args!(
822 "fallback database cleanup could not be queued: {error}"
823 ));
824 }
825 }
826}
827
828struct DatabaseLeaseInner {
829 server: Arc<ServerInner>,
830 name: DatabaseName,
831 permit: OwnedSemaphorePermit,
832 prewarm_pool: Option<Weak<PrewarmPoolInner>>,
833}
834
835impl DatabaseLeaseInner {
836 fn queue_awaited_cleanup(
837 self,
838 ) -> Result<tokio::sync::oneshot::Receiver<crate::cleanup::AwaitedCleanupOutcome>> {
839 let Self {
840 server,
841 name,
842 permit,
843 prewarm_pool,
844 } = self;
845 let database_name = name.as_str().to_owned();
846 let database_cleanup = server.database_cleanup.clone();
847 let prewarm_return = prewarm_pool
848 .and_then(|pool| pool.upgrade())
849 .map(|pool| pool.begin_return());
850 database_cleanup.submit_awaited_outcome(
851 database_name,
852 permit,
853 database_cleanup_operation(server, name, prewarm_return),
854 )
855 }
856
857 fn queue_deferred_cleanup(self) -> Result<()> {
858 let Self {
859 server,
860 name,
861 permit,
862 prewarm_pool,
863 } = self;
864 let database_name = name.as_str().to_owned();
865 let database_cleanup = server.database_cleanup.clone();
866 let prewarm_return = prewarm_pool
867 .and_then(|pool| pool.upgrade())
868 .map(|pool| pool.begin_return());
869 database_cleanup.submit_deferred_outcome(
870 database_name,
871 Some(permit),
872 database_cleanup_operation(server, name, prewarm_return),
873 )
874 }
875
876 fn queue_fallback_cleanup(self) -> Result<()> {
877 let Self {
878 server,
879 name,
880 permit,
881 prewarm_pool,
882 } = self;
883 let database_name = name.as_str().to_owned();
884 let database_cleanup = server.database_cleanup.clone();
885 let prewarm_return = prewarm_pool
886 .and_then(|pool| pool.upgrade())
887 .map(|pool| pool.begin_return());
888 database_cleanup.submit_fallback_outcome(
889 database_name,
890 permit,
891 database_cleanup_operation(server, name, prewarm_return),
892 )
893 }
894}
895
896fn database_cleanup_operation(
897 server: Arc<ServerInner>,
898 name: DatabaseName,
899 mut prewarm_return: Option<PrewarmReturn>,
900) -> impl FnOnce() -> CleanupOutcome + Send + 'static {
901 move || {
902 let operation_server = server.clone();
903 let outcome = server.with_lifecycle_admin_disposition(
904 "connect for disposable database cleanup",
905 move |client| {
906 if let Err(error) = drop_database(client, &name) {
907 return AdminSessionDisposition::Evict(CleanupOutcome::residual_possible(
908 error,
909 ));
910 }
911 let Some(ref mut refill) = prewarm_return else {
912 return AdminSessionDisposition::Reuse(CleanupOutcome::succeeded());
913 };
914 if !refill.after_drop() {
915 return AdminSessionDisposition::Reuse(CleanupOutcome::succeeded());
916 }
917
918 let fresh_name = DatabaseName::test(&operation_server.project);
919 if let Err(error) = operation_server.create_managed_database_classified(
920 client,
921 &fresh_name,
922 refill.pool.template.name().as_str(),
923 &ResourceMetadata::test(
924 operation_server.project.clone(),
925 operation_server.owner_key,
926 ),
927 ) {
928 return match error {
929 ManagedDatabaseCreationFailure::NoResidual(error) => {
930 AdminSessionDisposition::Reuse(CleanupOutcome::no_residual(error))
931 }
932 ManagedDatabaseCreationFailure::ResidualPossible(error) => {
933 AdminSessionDisposition::Evict(CleanupOutcome::residual_possible(error))
934 }
935 };
936 }
937 let prepared = PreparedDatabase {
938 database_url: operation_server.admin_url.database_url(&fresh_name),
939 name: fresh_name,
940 };
941 if let Some((prepared, deletion)) = refill.publish(prepared) {
942 let result = drop_database(client, &prepared.name);
946 deletion.finish();
947 if let Err(error) = result {
948 return AdminSessionDisposition::Evict(CleanupOutcome::residual_possible(
949 error,
950 ));
951 }
952 }
953 AdminSessionDisposition::Reuse(CleanupOutcome::succeeded())
954 },
955 );
956 match outcome {
957 Ok(outcome) => outcome,
958 Err(error) => CleanupOutcome::residual_possible(error),
959 }
960 }
961}
962
963enum TemplatePreparation {
964 Ready(PersistentClient),
965 Initialize(TemplateInitialization),
966}
967
968struct TemplateInitialization {
969 server: Arc<ServerInner>,
970 name: DatabaseName,
971 fingerprint: TemplateFingerprint,
972 lock_key: i64,
973 coordination_key: i64,
974 exclusive_lock: PersistentClient,
975}
976
977impl TemplateInitialization {
978 fn finish(self) -> Result<PersistentClient> {
979 let Self {
980 server,
981 name,
982 fingerprint,
983 lock_key,
984 coordination_key,
985 mut exclusive_lock,
986 } = self;
987 let client = exclusive_lock.client_mut();
988 disable_database_connections(client, &name)?;
989 terminate_database_connections(client, &name, server.operation_timeout)?;
990 set_database_metadata(
991 client,
992 &name,
993 &ResourceMetadata::template(
994 server.project.clone(),
995 lock_key,
996 fingerprint,
997 TemplateState::Ready,
998 ),
999 )?;
1000 release_advisory_lock(client, lock_key)?;
1001 let ready = acquire_and_inspect_shared_template_lock(
1002 client,
1003 &server,
1004 &name,
1005 fingerprint,
1006 lock_key,
1007 )?;
1008 release_advisory_lock(client, coordination_key)?;
1009 if ready {
1010 Ok(exclusive_lock)
1011 } else {
1012 Err(Error::InconsistentMetadata {
1013 database_name: name.as_str().to_owned(),
1014 })
1015 }
1016 }
1017
1018 fn abort(self) -> Result<()> {
1019 let Self {
1020 name,
1021 lock_key,
1022 coordination_key,
1023 mut exclusive_lock,
1024 ..
1025 } = self;
1026 let client = exclusive_lock.client_mut();
1027 let cleanup = drop_database(client, &name);
1028 let unlock_template = release_advisory_lock(client, lock_key);
1029 let unlock_coordination = release_advisory_lock(client, coordination_key);
1030 cleanup.and(unlock_template).and(unlock_coordination)
1031 }
1032}
1033
1034fn begin_template(
1035 server: Arc<ServerInner>,
1036 name: DatabaseName,
1037 fingerprint: TemplateFingerprint,
1038) -> Result<TemplatePreparation> {
1039 let lock_key = template_lock_key(&server.project, fingerprint);
1040 let coordination_key = template_coordination_key(&server.project, fingerprint);
1041 let mut persistent_client = PersistentClient::new(connect_admin(
1042 &server.admin_url,
1043 server.operation_timeout,
1044 "connect for PostgreSQL template coordination",
1045 )?);
1046 loop {
1047 let client = persistent_client.client_mut();
1048 if acquire_and_inspect_shared_template_lock(client, &server, &name, fingerprint, lock_key)?
1049 {
1050 return Ok(TemplatePreparation::Ready(persistent_client));
1051 }
1052
1053 acquire_template_advisory_lock(
1056 client,
1057 coordination_key,
1058 server.template_wait_timeout,
1059 server.operation_timeout,
1060 )?;
1061 if acquire_and_inspect_shared_template_lock(client, &server, &name, fingerprint, lock_key)?
1062 {
1063 release_advisory_lock(client, coordination_key)?;
1064 return Ok(TemplatePreparation::Ready(persistent_client));
1065 }
1066
1067 acquire_template_advisory_lock(
1068 client,
1069 lock_key,
1070 server.template_wait_timeout,
1071 server.operation_timeout,
1072 )?;
1073 if template_is_ready(client, &server, &name, fingerprint, lock_key)? {
1074 release_advisory_lock(client, lock_key)?;
1075 let ready = acquire_and_inspect_shared_template_lock(
1076 client,
1077 &server,
1078 &name,
1079 fingerprint,
1080 lock_key,
1081 )?;
1082 release_advisory_lock(client, coordination_key)?;
1083 if ready {
1084 return Ok(TemplatePreparation::Ready(persistent_client));
1085 }
1086 continue;
1087 }
1088 if let Some(record) = find_database(client, &name)? {
1089 let recoverable = record
1090 .comment
1091 .as_deref()
1092 .and_then(ResourceMetadata::parse)
1093 .is_some_and(|metadata| {
1094 recoverable_template_initialization(
1095 &metadata,
1096 &server.project,
1097 fingerprint,
1098 lock_key,
1099 )
1100 });
1101 if !recoverable {
1102 return Err(Error::InconsistentMetadata {
1103 database_name: name.as_str().to_owned(),
1104 });
1105 }
1106 drop_database(client, &name)?;
1107 }
1108 server
1109 .create_managed_database_classified(
1110 client,
1111 &name,
1112 "template0",
1113 &ResourceMetadata::template(
1114 server.project.clone(),
1115 lock_key,
1116 fingerprint,
1117 TemplateState::Initializing,
1118 ),
1119 )
1120 .map_err(ManagedDatabaseCreationFailure::into_error)?;
1121 return Ok(TemplatePreparation::Initialize(TemplateInitialization {
1122 server,
1123 name,
1124 fingerprint,
1125 lock_key,
1126 coordination_key,
1127 exclusive_lock: persistent_client,
1128 }));
1129 }
1130}
1131
1132fn template_lock_key(project: &ProjectName, fingerprint: TemplateFingerprint) -> i64 {
1133 advisory_key(
1134 "template",
1135 &format!("{}:{}", project.as_str(), fingerprint.to_hex()),
1136 )
1137}
1138
1139fn template_coordination_key(project: &ProjectName, fingerprint: TemplateFingerprint) -> i64 {
1140 advisory_key(
1141 "template-coordination",
1142 &format!("{}:{}", project.as_str(), fingerprint.to_hex()),
1143 )
1144}
1145
1146fn acquire_and_inspect_shared_template_lock(
1147 client: &mut AdminClient,
1148 server: &ServerInner,
1149 name: &DatabaseName,
1150 fingerprint: TemplateFingerprint,
1151 lock_key: i64,
1152) -> Result<bool> {
1153 acquire_shared_template_advisory_lock(
1154 client,
1155 lock_key,
1156 server.template_wait_timeout,
1157 server.operation_timeout,
1158 )?;
1159 if template_is_ready(client, server, name, fingerprint, lock_key)? {
1160 Ok(true)
1161 } else {
1162 release_shared_advisory_lock(client, lock_key)?;
1163 Ok(false)
1164 }
1165}
1166
1167fn recoverable_template_initialization(
1168 metadata: &ResourceMetadata,
1169 project: &ProjectName,
1170 fingerprint: TemplateFingerprint,
1171 lock_key: i64,
1172) -> bool {
1173 matches!(
1174 metadata,
1175 ResourceMetadata::Template {
1176 project: metadata_project,
1177 lock_key: metadata_lock_key,
1178 fingerprint: metadata_fingerprint,
1179 state: TemplateState::Initializing,
1180 ..
1181 } if metadata_project == project
1182 && *metadata_lock_key == lock_key
1183 && *metadata_fingerprint == fingerprint
1184 )
1185}
1186
1187fn template_is_ready(
1188 client: &mut AdminClient,
1189 server: &ServerInner,
1190 name: &DatabaseName,
1191 fingerprint: TemplateFingerprint,
1192 lock_key: i64,
1193) -> Result<bool> {
1194 let record = find_database(client, name)?;
1195 Ok(template_record_is_ready(
1196 record.as_ref(),
1197 &server.project,
1198 fingerprint,
1199 lock_key,
1200 ))
1201}
1202
1203fn template_record_is_ready(
1204 record: Option<&DatabaseRecord>,
1205 project: &ProjectName,
1206 fingerprint: TemplateFingerprint,
1207 lock_key: i64,
1208) -> bool {
1209 let Some(comment) = record.and_then(|record| record.comment.as_deref()) else {
1210 return false;
1211 };
1212 matches!(
1213 ResourceMetadata::parse(comment),
1214 Some(ResourceMetadata::Template {
1215 project: metadata_project,
1216 lock_key: metadata_lock_key,
1217 fingerprint: metadata_fingerprint,
1218 state: TemplateState::Ready,
1219 ..
1220 }) if metadata_project == *project
1221 && metadata_lock_key == lock_key
1222 && metadata_fingerprint == fingerprint
1223 )
1224}
1225
1226async fn create_test_database(
1227 server: Arc<ServerInner>,
1228 template_name: &str,
1229) -> Result<DatabaseLease> {
1230 let permit = server.acquire_database_permit().await?;
1231 let template_name = template_name.to_owned();
1232 let name = DatabaseName::test(&server.project);
1233 let database_url = server.admin_url.database_url(&name);
1234 run_blocking(move || {
1235 server.with_lifecycle_admin("connect for disposable database creation", |client| {
1236 server
1237 .create_managed_database_classified(
1238 client,
1239 &name,
1240 &template_name,
1241 &ResourceMetadata::test(server.project.clone(), server.owner_key),
1242 )
1243 .map_err(ManagedDatabaseCreationFailure::into_error)
1244 })?;
1245 Ok(DatabaseLease {
1246 inner: Some(DatabaseLeaseInner {
1247 server,
1248 name,
1249 permit,
1250 prewarm_pool: None,
1251 }),
1252 database_url,
1253 })
1254 })
1255 .await
1256}
1257
1258async fn create_unpublished_database(
1259 server: Arc<ServerInner>,
1260 template_name: String,
1261) -> Result<UnpublishedDatabase> {
1262 run_blocking(move || {
1263 let name = DatabaseName::test(&server.project);
1264 let database_url = server.admin_url.database_url(&name);
1265 server.with_lifecycle_admin("connect for prewarmed database creation", |client| {
1266 server
1267 .create_managed_database_classified(
1268 client,
1269 &name,
1270 &template_name,
1271 &ResourceMetadata::test(server.project.clone(), server.owner_key),
1272 )
1273 .map_err(ManagedDatabaseCreationFailure::into_error)
1274 })?;
1275 Ok(UnpublishedDatabase {
1276 server,
1277 prepared: Some(PreparedDatabase { name, database_url }),
1278 })
1279 })
1280 .await
1281}
1282
1283fn queue_unpublished_database(server: Arc<ServerInner>, prepared: PreparedDatabase) {
1284 let database_name = prepared.name.as_str().to_owned();
1285 let database_cleanup = server.database_cleanup.clone();
1286 if let Err(error) = database_cleanup.submit_prewarmer(database_name, move || {
1287 server.with_lifecycle_admin("connect for unpublished prewarm cleanup", |client| {
1288 drop_database(client, &prepared.name)
1289 })
1290 }) {
1291 log_cleanup(format_args!(
1294 "unpublished prewarmed database cleanup could not be queued: {error}"
1295 ));
1296 }
1297}
1298
1299fn queue_closed_prepared_database(pool: Arc<PrewarmPoolInner>, prepared: PreparedDatabase) {
1300 let server = pool.template.server().clone();
1301 let deletion = ClosedPrewarmDeletion { pool, active: true };
1302 queue_counted_prepared_database(server, prepared, deletion);
1303}
1304
1305fn queue_counted_prepared_database(
1306 server: Arc<ServerInner>,
1307 prepared: PreparedDatabase,
1308 deletion: ClosedPrewarmDeletion,
1309) {
1310 let database_name = prepared.name.as_str().to_owned();
1311 let database_cleanup = server.database_cleanup.clone();
1312 if let Err(error) = database_cleanup.submit_prewarmer(database_name, move || {
1313 let result = server.with_lifecycle_admin("connect for prewarm pool shutdown", |client| {
1314 drop_database(client, &prepared.name)
1315 });
1316 deletion.finish();
1317 result
1318 }) {
1319 log_cleanup(format_args!(
1320 "idle prewarmed database cleanup could not be queued: {error}"
1321 ));
1322 }
1323}
1324
1325#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
1327#[non_exhaustive]
1328pub struct CleanupReport {
1329 pub dropped_test_databases: usize,
1330 pub dropped_templates: usize,
1331 pub skipped_active: usize,
1332 pub skipped_fresh: usize,
1333 pub skipped_unrecognized: usize,
1334}
1335
1336impl CleanupReport {
1337 pub fn total_dropped(self) -> usize {
1338 self.dropped_test_databases + self.dropped_templates
1339 }
1340}
1341
1342pub async fn cleanup_stale_databases(
1344 admin_database_url: &str,
1345 project: impl Into<String>,
1346 stale_after: Duration,
1347) -> Result<CleanupReport> {
1348 let admin_url = AdminDatabaseUrl::parse(admin_database_url)?;
1349 let project = ProjectName::new(project)?;
1350 cleanup_stale_with(
1351 admin_url,
1352 project,
1353 stale_after,
1354 DEFAULT_CLEANUP_OPERATION_TIMEOUT,
1355 )
1356 .await
1357}
1358
1359async fn cleanup_stale_with(
1360 admin_url: AdminDatabaseUrl,
1361 project: ProjectName,
1362 stale_after: Duration,
1363 operation_timeout: Duration,
1364) -> Result<CleanupReport> {
1365 run_blocking(move || {
1366 cleanup_stale_blocking(&admin_url, &project, stale_after, operation_timeout)
1367 })
1368 .await
1369}
1370
1371fn cleanup_stale_blocking(
1372 admin_url: &AdminDatabaseUrl,
1373 project: &ProjectName,
1374 stale_after: Duration,
1375 operation_timeout: Duration,
1376) -> Result<CleanupReport> {
1377 let mut client = connect_admin(
1378 admin_url,
1379 operation_timeout,
1380 "connect for stale PostgreSQL database cleanup",
1381 )?;
1382 validate_postgres_18(&mut client)?;
1383 let records = list_databases(&mut client)?;
1384 let now = SystemTime::now()
1385 .duration_since(SystemTime::UNIX_EPOCH)
1386 .unwrap_or_default()
1387 .as_secs();
1388 let mut report = CleanupReport::default();
1389
1390 for record in records {
1391 let lock_key = match classify_cleanup_record(project, &record, now, stale_after) {
1392 CleanupClassification::Ignore => continue,
1393 CleanupClassification::Unrecognized => {
1394 report.skipped_unrecognized += 1;
1395 continue;
1396 }
1397 CleanupClassification::Fresh => {
1398 report.skipped_fresh += 1;
1399 continue;
1400 }
1401 CleanupClassification::Candidate { lock_key, .. } => lock_key,
1402 };
1403 if !try_acquire_advisory_lock(&mut client, lock_key)? {
1404 report.skipped_active += 1;
1405 continue;
1406 }
1407 let cleanup_result = cleanup_candidate_after_lock(
1408 &mut client,
1409 project,
1410 &record,
1411 lock_key,
1412 now,
1413 stale_after,
1414 &mut report,
1415 );
1416 let Some(dropped_kind) = finish_locked_cleanup(cleanup_result, || {
1417 release_advisory_lock(&mut client, lock_key)
1418 })?
1419 else {
1420 continue;
1421 };
1422 match dropped_kind {
1423 DatabaseKind::Test => report.dropped_test_databases += 1,
1424 DatabaseKind::Template => report.dropped_templates += 1,
1425 }
1426 }
1427 Ok(report)
1428}
1429
1430fn cleanup_candidate_after_lock(
1431 client: &mut AdminClient,
1432 project: &ProjectName,
1433 snapshot: &DatabaseRecord,
1434 held_key: i64,
1435 now: u64,
1436 stale_after: Duration,
1437 report: &mut CleanupReport,
1438) -> Result<Option<DatabaseKind>> {
1439 let snapshot_name = DatabaseName::from_existing(project, snapshot.name.clone())
1440 .expect("classified cleanup candidate has a valid managed database name");
1441 let current = find_database(client, &snapshot_name)?;
1442 let Some(name) = revalidate_cleanup_candidate(
1443 project,
1444 snapshot,
1445 current.as_ref(),
1446 held_key,
1447 now,
1448 stale_after,
1449 report,
1450 ) else {
1451 return Ok(None);
1452 };
1453 drop_database(client, &name)?;
1454 Ok(Some(name.kind()))
1455}
1456
1457fn finish_locked_cleanup<T>(
1458 operation: Result<T>,
1459 unlock: impl FnOnce() -> Result<()>,
1460) -> Result<T> {
1461 let unlock = unlock();
1462 match operation {
1463 Ok(value) => {
1464 unlock?;
1465 Ok(value)
1466 }
1467 Err(error) => Err(error),
1468 }
1469}
1470
1471enum CleanupClassification {
1472 Ignore,
1473 Unrecognized,
1474 Fresh,
1475 Candidate { name: DatabaseName, lock_key: i64 },
1476}
1477
1478fn classify_cleanup_record(
1479 project: &ProjectName,
1480 record: &DatabaseRecord,
1481 now: u64,
1482 stale_after: Duration,
1483) -> CleanupClassification {
1484 let Some(name) = DatabaseName::from_existing(project, record.name.clone()) else {
1485 return CleanupClassification::Ignore;
1486 };
1487 let Some(metadata) = record.comment.as_deref().and_then(ResourceMetadata::parse) else {
1488 return CleanupClassification::Unrecognized;
1489 };
1490 if metadata.project() != project || !metadata_matches_name(&metadata, &name) {
1491 return CleanupClassification::Unrecognized;
1492 }
1493 if now.saturating_sub(metadata.created_at()) < stale_after.as_secs() {
1494 return CleanupClassification::Fresh;
1495 }
1496 CleanupClassification::Candidate {
1497 name,
1498 lock_key: metadata.lock_key(),
1499 }
1500}
1501
1502fn revalidate_cleanup_candidate(
1503 project: &ProjectName,
1504 snapshot: &DatabaseRecord,
1505 current: Option<&DatabaseRecord>,
1506 held_key: i64,
1507 now: u64,
1508 stale_after: Duration,
1509 report: &mut CleanupReport,
1510) -> Option<DatabaseName> {
1511 let current = current?;
1512 match classify_cleanup_record(project, current, now, stale_after) {
1513 CleanupClassification::Ignore => None,
1514 CleanupClassification::Unrecognized => {
1515 report.skipped_unrecognized += 1;
1516 None
1517 }
1518 CleanupClassification::Fresh => {
1519 report.skipped_fresh += 1;
1520 None
1521 }
1522 CleanupClassification::Candidate { name, lock_key }
1523 if current == snapshot && lock_key == held_key =>
1524 {
1525 Some(name)
1526 }
1527 CleanupClassification::Candidate { .. } => None,
1528 }
1529}
1530
1531fn metadata_matches_name(metadata: &ResourceMetadata, name: &DatabaseName) -> bool {
1532 match (metadata, name.kind()) {
1533 (ResourceMetadata::Test { .. }, DatabaseKind::Test) => true,
1534 (
1535 ResourceMetadata::Template {
1536 project,
1537 fingerprint,
1538 ..
1539 },
1540 DatabaseKind::Template,
1541 ) => DatabaseName::template(project, *fingerprint) == *name,
1542 _ => false,
1543 }
1544}
1545
1546#[cfg(test)]
1547mod tests {
1548 use std::{
1549 sync::{
1550 Arc,
1551 atomic::{AtomicBool, Ordering},
1552 mpsc,
1553 },
1554 time::Duration,
1555 };
1556
1557 use tokio::sync::{Semaphore, oneshot};
1558
1559 use super::{
1560 CleanupClassification, CleanupReport, PreparedDatabase, PrewarmPoolState,
1561 PrewarmReturnPhase, acquire_database_permit_or_pool_close, classify_cleanup_record,
1562 finish_locked_cleanup, recoverable_template_initialization, revalidate_cleanup_candidate,
1563 run_blocking, template_coordination_key, template_lock_key, template_record_is_ready,
1564 };
1565 use crate::{
1566 Error, FingerprintBuilder, ProjectName,
1567 admin::DatabaseRecord,
1568 metadata::{ResourceMetadata, TemplateState},
1569 name::{DatabaseKind, DatabaseName},
1570 };
1571
1572 struct DropProbe(Arc<AtomicBool>);
1573
1574 impl Drop for DropProbe {
1575 fn drop(&mut self) {
1576 self.0.store(true, Ordering::SeqCst);
1577 }
1578 }
1579
1580 #[tokio::test]
1581 async fn prewarm_close_wakes_database_capacity_wait() {
1582 let database_admission = Arc::new(Semaphore::new(0));
1583 let pool_closed = Arc::new(Semaphore::new(0));
1584 let (started_sender, started_receiver) = oneshot::channel();
1585 let waiting_admission = database_admission.clone();
1586 let waiting_close = pool_closed.clone();
1587 let waiter = tokio::spawn(async move {
1588 acquire_database_permit_or_pool_close(
1589 async move {
1590 let _ = started_sender.send(());
1591 waiting_admission
1592 .acquire_owned()
1593 .await
1594 .map_err(|_| Error::ConnectionBudgetClosed)
1595 },
1596 waiting_close,
1597 )
1598 .await
1599 });
1600 started_receiver
1601 .await
1602 .expect("database-capacity wait should start");
1603
1604 pool_closed.close();
1605 let error = tokio::time::timeout(Duration::from_secs(1), waiter)
1606 .await
1607 .expect("pool closure should wake the database-capacity wait")
1608 .expect("capacity waiter should not panic")
1609 .expect_err("a closed prewarm pool must reject the checkout");
1610 assert!(matches!(error, Error::PrewarmPoolClosed));
1611 }
1612
1613 #[test]
1614 fn failed_prewarm_phase_begins_cleanup_for_every_ready_database() {
1615 let project = ProjectName::new("prewarm_failure").unwrap();
1616 let mut state = PrewarmPoolState::new();
1617 state.ready.extend((0..2).map(|_| {
1618 let name = DatabaseName::test(&project);
1619 PreparedDatabase {
1620 database_url: format!("postgres://localhost/{}", name.as_str()),
1621 name,
1622 }
1623 }));
1624 state.creating = 1;
1625
1626 let ready = state
1627 .fail_and_begin_cleanup(PrewarmReturnPhase::Creating, 3)
1628 .expect("the first failure starts terminal cleanup");
1629
1630 assert_eq!(ready.len(), 2);
1631 assert!(state.ready.is_empty());
1632 assert_eq!(state.creating, 0);
1633 assert_eq!(state.deleting, 2);
1634 assert_eq!(state.occupied_slots(), 2);
1635 assert!(!state.accepting);
1636 assert!(state.cleanup_started);
1637 assert!(state.begin_cleanup(3).is_none());
1638 }
1639
1640 fn test_record(
1641 project: &ProjectName,
1642 name: &DatabaseName,
1643 owner_key: i64,
1644 created_at: u64,
1645 ) -> DatabaseRecord {
1646 DatabaseRecord {
1647 name: name.as_str().to_owned(),
1648 comment: Some(
1649 ResourceMetadata::Test {
1650 project: project.clone(),
1651 owner_key,
1652 created_at,
1653 }
1654 .encode(),
1655 ),
1656 }
1657 }
1658
1659 fn template_record(
1660 project: &ProjectName,
1661 fingerprint: crate::TemplateFingerprint,
1662 lock_key: i64,
1663 state: TemplateState,
1664 created_at: u64,
1665 ) -> DatabaseRecord {
1666 DatabaseRecord {
1667 name: DatabaseName::template(project, fingerprint)
1668 .as_str()
1669 .to_owned(),
1670 comment: Some(
1671 ResourceMetadata::Template {
1672 project: project.clone(),
1673 lock_key,
1674 fingerprint,
1675 state,
1676 created_at,
1677 }
1678 .encode(),
1679 ),
1680 }
1681 }
1682
1683 #[tokio::test(flavor = "multi_thread")]
1684 async fn detached_blocking_results_are_dropped() {
1685 let cleaned = Arc::new(AtomicBool::new(false));
1686 let cleaned_for_task = cleaned.clone();
1687 let (started_sender, started_receiver) = oneshot::channel();
1688 let (release_sender, release_receiver) = mpsc::channel();
1689
1690 let task = tokio::spawn(async move {
1691 run_blocking(move || {
1692 let _ = started_sender.send(());
1693 release_receiver.recv().unwrap();
1694 Ok(DropProbe(cleaned_for_task))
1695 })
1696 .await
1697 });
1698
1699 started_receiver.await.unwrap();
1700 task.abort();
1701 release_sender.send(()).unwrap();
1702 let join_error = match task.await {
1703 Ok(_) => panic!("cancelled task unexpectedly completed"),
1704 Err(error) => error,
1705 };
1706 assert!(join_error.is_cancelled());
1707 tokio::time::timeout(Duration::from_secs(1), async {
1708 while !cleaned.load(Ordering::SeqCst) {
1709 tokio::task::yield_now().await;
1710 }
1711 })
1712 .await
1713 .expect("detached blocking result should be cleaned");
1714 }
1715
1716 #[test]
1717 fn template_lock_domains_are_distinct_and_namespaced_by_project() {
1718 let fingerprint = FingerprintBuilder::new("schema").finish();
1719 let left = template_lock_key(&ProjectName::new("left").unwrap(), fingerprint);
1720 let right = template_lock_key(&ProjectName::new("right").unwrap(), fingerprint);
1721 let coordination =
1722 template_coordination_key(&ProjectName::new("left").unwrap(), fingerprint);
1723 assert_ne!(left, right);
1724 assert_ne!(left, coordination);
1725 }
1726
1727 #[test]
1728 fn only_matching_initializing_templates_are_recoverable() {
1729 let project = ProjectName::new("creditkit").unwrap();
1730 let fingerprint = FingerprintBuilder::new("schema").finish();
1731 let lock_key = template_lock_key(&project, fingerprint);
1732 let initializing = ResourceMetadata::template(
1733 project.clone(),
1734 lock_key,
1735 fingerprint,
1736 TemplateState::Initializing,
1737 );
1738 let ready = ResourceMetadata::template(
1739 project.clone(),
1740 lock_key,
1741 fingerprint,
1742 TemplateState::Ready,
1743 );
1744 assert!(recoverable_template_initialization(
1745 &initializing,
1746 &project,
1747 fingerprint,
1748 lock_key
1749 ));
1750 assert!(!recoverable_template_initialization(
1751 &ready,
1752 &project,
1753 fingerprint,
1754 lock_key
1755 ));
1756 assert!(!recoverable_template_initialization(
1757 &initializing,
1758 &ProjectName::new("another").unwrap(),
1759 fingerprint,
1760 lock_key
1761 ));
1762 }
1763
1764 #[test]
1765 fn exact_template_records_preserve_catalog_states() {
1766 let project = ProjectName::new("creditkit").unwrap();
1767 let fingerprint = FingerprintBuilder::new("schema").finish();
1768 let lock_key = template_lock_key(&project, fingerprint);
1769 let name = DatabaseName::template(&project, fingerprint)
1770 .as_str()
1771 .to_owned();
1772 let untagged = DatabaseRecord {
1773 name: name.clone(),
1774 comment: None,
1775 };
1776 let initializing = DatabaseRecord {
1777 name: name.clone(),
1778 comment: Some(
1779 ResourceMetadata::template(
1780 project.clone(),
1781 lock_key,
1782 fingerprint,
1783 TemplateState::Initializing,
1784 )
1785 .encode(),
1786 ),
1787 };
1788 let ready = DatabaseRecord {
1789 name,
1790 comment: Some(
1791 ResourceMetadata::template(
1792 project.clone(),
1793 lock_key,
1794 fingerprint,
1795 TemplateState::Ready,
1796 )
1797 .encode(),
1798 ),
1799 };
1800
1801 assert!(!template_record_is_ready(
1802 None,
1803 &project,
1804 fingerprint,
1805 lock_key
1806 ));
1807 assert!(!template_record_is_ready(
1808 Some(&untagged),
1809 &project,
1810 fingerprint,
1811 lock_key
1812 ));
1813 assert!(!template_record_is_ready(
1814 Some(&initializing),
1815 &project,
1816 fingerprint,
1817 lock_key
1818 ));
1819 assert!(template_record_is_ready(
1820 Some(&ready),
1821 &project,
1822 fingerprint,
1823 lock_key
1824 ));
1825 }
1826
1827 #[test]
1828 fn cleanup_classification_is_conservative_without_postgres() {
1829 let project = ProjectName::new("creditkit").unwrap();
1830 let test_name = DatabaseName::test(&project);
1831 let metadata = ResourceMetadata::test(project.clone(), 42);
1832 let created_at = metadata.created_at();
1833
1834 let unrecognized = classify_cleanup_record(
1835 &project,
1836 &DatabaseRecord {
1837 name: test_name.as_str().to_owned(),
1838 comment: None,
1839 },
1840 created_at + 100,
1841 Duration::from_secs(10),
1842 );
1843 assert!(matches!(unrecognized, CleanupClassification::Unrecognized));
1844
1845 let fresh = classify_cleanup_record(
1846 &project,
1847 &DatabaseRecord {
1848 name: test_name.as_str().to_owned(),
1849 comment: Some(metadata.encode()),
1850 },
1851 created_at + 5,
1852 Duration::from_secs(10),
1853 );
1854 assert!(matches!(fresh, CleanupClassification::Fresh));
1855
1856 let candidate = classify_cleanup_record(
1857 &project,
1858 &DatabaseRecord {
1859 name: test_name.as_str().to_owned(),
1860 comment: Some(metadata.encode()),
1861 },
1862 created_at + 10,
1863 Duration::from_secs(10),
1864 );
1865 assert!(matches!(
1866 candidate,
1867 CleanupClassification::Candidate { lock_key: 42, .. }
1868 ));
1869
1870 let malformed_name = classify_cleanup_record(
1871 &project,
1872 &DatabaseRecord {
1873 name: "pgh_creditkit_test_backup".to_owned(),
1874 comment: Some(metadata.encode()),
1875 },
1876 created_at + 10,
1877 Duration::from_secs(10),
1878 );
1879 assert!(matches!(malformed_name, CleanupClassification::Ignore));
1880 }
1881
1882 #[test]
1883 fn locked_cleanup_requires_an_unchanged_candidate() {
1884 let project = ProjectName::new("creditkit").unwrap();
1885 let test_name = DatabaseName::test(&project);
1886 let snapshot = test_record(&project, &test_name, 42, 10);
1887 let mut report = CleanupReport::default();
1888
1889 let unchanged = revalidate_cleanup_candidate(
1890 &project,
1891 &snapshot,
1892 Some(&snapshot),
1893 42,
1894 100,
1895 Duration::from_secs(10),
1896 &mut report,
1897 );
1898 assert_eq!(
1899 unchanged.as_ref().map(DatabaseName::kind),
1900 Some(DatabaseKind::Test)
1901 );
1902
1903 let refreshed = test_record(&project, &test_name, 42, 95);
1904 assert!(
1905 revalidate_cleanup_candidate(
1906 &project,
1907 &snapshot,
1908 Some(&refreshed),
1909 42,
1910 100,
1911 Duration::from_secs(10),
1912 &mut report,
1913 )
1914 .is_none()
1915 );
1916 assert_eq!(report.skipped_fresh, 1);
1917
1918 let changed_key = test_record(&project, &test_name, 84, 10);
1919 assert!(
1920 revalidate_cleanup_candidate(
1921 &project,
1922 &snapshot,
1923 Some(&changed_key),
1924 42,
1925 100,
1926 Duration::from_secs(10),
1927 &mut report,
1928 )
1929 .is_none()
1930 );
1931 assert!(
1932 revalidate_cleanup_candidate(
1933 &project,
1934 &snapshot,
1935 None,
1936 42,
1937 100,
1938 Duration::from_secs(10),
1939 &mut report,
1940 )
1941 .is_none()
1942 );
1943 }
1944
1945 #[test]
1946 fn locked_cleanup_rejects_a_finalized_template_snapshot() {
1947 let project = ProjectName::new("creditkit").unwrap();
1948 let fingerprint = FingerprintBuilder::new("cleanup-race").finish();
1949 let initializing =
1950 template_record(&project, fingerprint, 42, TemplateState::Initializing, 10);
1951 let ready = template_record(&project, fingerprint, 42, TemplateState::Ready, 10);
1952 let mut report = CleanupReport::default();
1953
1954 assert!(
1955 revalidate_cleanup_candidate(
1956 &project,
1957 &initializing,
1958 Some(&ready),
1959 42,
1960 100,
1961 Duration::from_secs(10),
1962 &mut report,
1963 )
1964 .is_none()
1965 );
1966 assert_eq!(report, CleanupReport::default());
1967 }
1968
1969 #[test]
1970 fn locked_cleanup_keeps_the_operation_error_primary() {
1971 let unlocked = AtomicBool::new(false);
1972 let error = finish_locked_cleanup::<()>(
1973 Err(Error::InvalidConfiguration {
1974 reason: "locked reread",
1975 }),
1976 || {
1977 unlocked.store(true, Ordering::SeqCst);
1978 Err(Error::InvalidConfiguration { reason: "unlock" })
1979 },
1980 )
1981 .unwrap_err();
1982 assert!(unlocked.load(Ordering::SeqCst));
1983 assert!(matches!(
1984 error,
1985 Error::InvalidConfiguration {
1986 reason: "locked reread"
1987 }
1988 ));
1989
1990 let error = finish_locked_cleanup(Ok(()), || {
1991 Err(Error::InvalidConfiguration { reason: "unlock" })
1992 })
1993 .unwrap_err();
1994 assert!(matches!(
1995 error,
1996 Error::InvalidConfiguration { reason: "unlock" }
1997 ));
1998 }
1999}