1use std::{fmt, time::Duration};
2
3use chrono::{DateTime, Utc};
4use sqlx::{PgConnection, PgPool};
5use syrup_rail::{
6 ApprovedProcessorEvidence, BillingEvent, BillingEventSubject, ChargeHostTarget,
7 GatewayMutationError, GatewayNotSubmittedError, GatewayPaymentOutcome, GatewayPaymentStatus,
8 GatewayProviderKey, GatewaySaleIntent, GatewaySaleRequest, HostChargePaymentResult,
9 HostChargePaymentResultBuildError, HostChargeReservation, HostChargeTargetTransition,
10 HostChargeTargetTransitionKind, HostChargeTargetTransitionOutcome, PaymentAttempt,
11 PaymentAttemptStatus, PaymentResolutionCode, ProcessorChargeProgression, ProcessorChargeRole,
12 ProcessorEvidence,
13};
14use thiserror::Error;
15
16use crate::host_charges::host_charge_attempt_matches_reservation;
17use crate::{
18 BillingTransactionCoordinator, BillingTransactionError, BillingTransactionSubjectState,
19 HostChargeStoreError, HostChargeSubmissionOutcome, HostChargeTargetError,
20 HostChargeTargetStore, ModeVerifiedGateway, ProcessorChargeStoreError,
21 admit_host_charge_submission_in_transaction,
22 attempts::{
23 AttemptApproval, AttemptResolutionStatus, AttemptTransition, PaymentAttemptStoreError,
24 find_payment_attempt_by_id_on_connection, lock_payment_attempt_by_id_on_connection,
25 persist_attempt_transition,
26 },
27 enrollment_application::{
28 GatewayNotSubmittedPolicy, OutcomeResolutionBoundary, PreparedAttemptReplay,
29 RateLimitCooldown, RateLimitCooldownCommitError, RateLimitCooldownOperation,
30 SubscriptionEnrollmentApplicationError, commit_rate_limit_cooldown_for_operation,
31 map_attempt_transition_error, mutation_error_evidence, park_locked_attempt,
32 restore_prepared_attempt_submission, set_application_timeouts,
33 should_surface_not_submitted_application,
34 },
35 processor_charges::{ObservedCharge, observe_processor_charge, transition_charge},
36};
37
38const BILLING_LOCK_TIMEOUT: Duration = Duration::from_millis(250);
39const APPROVED_APPLICATION_ATTEMPTS: usize = 3;
40const APPROVED_EVIDENCE_RETRY_DELAY: Duration = Duration::from_millis(50);
41const INVALID_HOST_CHARGE_STATE: &str = "canonical host charge application state is invalid";
42const APPROVED_STALE_TARGET_TEXT: &str =
43 "Approved host charge could not update its target because the target changed.";
44const APPROVED_STORAGE_FAILURE_TEXT: &str =
45 "Approved host charge could not be applied; manual review is required.";
46
47#[derive(Debug, Error)]
48pub enum HostChargeApplicationError {
49 #[error("host charge application storage failed")]
50 Sql(#[from] sqlx::Error),
51 #[error("host charge attempt storage failed")]
52 Attempt(#[from] PaymentAttemptStoreError),
53 #[error("host charge target operation failed")]
54 Target(#[from] HostChargeTargetError),
55 #[error("host charge storage operation failed")]
56 Store(#[from] HostChargeStoreError),
57 #[error("host charge processor evidence operation failed")]
58 ProcessorCharge(#[from] ProcessorChargeStoreError),
59 #[error("host billing transaction failed")]
60 Transaction(#[from] BillingTransactionError),
61 #[error("host billing event append failed")]
62 Event(#[from] crate::BillingEventWriteError),
63 #[error("shared payment application failed")]
64 SharedApplication(#[from] SubscriptionEnrollmentApplicationError),
65 #[error("admitted host charge does not match the submission command or gateway")]
66 SubmissionIdentityMismatch,
67 #[error("{0}")]
68 InvalidState(&'static str),
69}
70
71impl From<HostChargePaymentResultBuildError> for HostChargeApplicationError {
72 fn from(_: HostChargePaymentResultBuildError) -> Self {
73 Self::InvalidState(INVALID_HOST_CHARGE_STATE)
74 }
75}
76
77pub struct AdmittedHostCharge {
78 reservation: HostChargeReservation,
79 attempt: PaymentAttempt,
80}
81
82impl AdmittedHostCharge {
83 pub const fn attempt(&self) -> &PaymentAttempt {
84 &self.attempt
85 }
86}
87
88impl fmt::Debug for AdmittedHostCharge {
89 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
90 formatter
91 .debug_struct("AdmittedHostCharge")
92 .field("attempt", &self.attempt)
93 .field("has_submission_authority", &true)
94 .finish()
95 }
96}
97
98#[derive(Debug)]
99pub enum HostChargeAdmissionOutcome {
100 Admitted(Box<AdmittedHostCharge>),
101 AlreadyAdmitted(PaymentAttempt),
102 Rejected {
103 attempt: PaymentAttempt,
104 reason: syrup_rail::HostChargeTargetRejection,
105 },
106}
107
108#[derive(Debug)]
109pub enum HostChargeProviderResult {
110 Payment(HostChargePaymentResult),
111 NotSubmitted {
115 payment: HostChargePaymentResult,
116 error: GatewayNotSubmittedError,
117 },
118}
119
120struct HostChargeResolutionApplication {
121 payment: HostChargePaymentResult,
122 applied: bool,
123}
124
125impl HostChargeResolutionApplication {
126 fn into_payment(self) -> HostChargePaymentResult {
127 self.payment
128 }
129
130 fn should_surface_not_submitted(&self, policy: GatewayNotSubmittedPolicy) -> bool {
131 should_surface_not_submitted_application(
132 self.applied,
133 self.payment.attempt(),
134 policy,
135 PreparedAttemptReplay::Supported,
136 )
137 }
138}
139
140#[derive(Clone, Copy)]
143pub(crate) struct HostChargeBeforeSubmissionResolution {
144 boundary: OutcomeResolutionBoundary,
145 cooldown: Option<RateLimitCooldown>,
146}
147
148#[derive(Clone, Copy)]
149struct HostChargeNonApprovedResolution<'provider> {
150 boundary: OutcomeResolutionBoundary,
151 cooldown: Option<(&'provider GatewayProviderKey, RateLimitCooldown)>,
152}
153
154impl HostChargeNonApprovedResolution<'_> {
155 const fn submitted() -> Self {
156 Self {
157 boundary: OutcomeResolutionBoundary::Submitted,
158 cooldown: None,
159 }
160 }
161}
162
163impl HostChargeBeforeSubmissionResolution {
164 pub(crate) const fn prepared() -> Self {
165 Self {
166 boundary: OutcomeResolutionBoundary::Prepared,
167 cooldown: None,
168 }
169 }
170
171 pub(crate) const fn admitted_not_submitted() -> Self {
172 Self {
173 boundary: OutcomeResolutionBoundary::AdmittedNotSubmitted,
174 cooldown: None,
175 }
176 }
177
178 pub(crate) const fn with_not_submitted_policy(self, policy: GatewayNotSubmittedPolicy) -> Self {
179 Self {
180 boundary: self.boundary,
181 cooldown: policy.cooldown(),
182 }
183 }
184
185 const fn with_provider(
186 self,
187 provider_key: &GatewayProviderKey,
188 ) -> HostChargeNonApprovedResolution<'_> {
189 HostChargeNonApprovedResolution {
190 boundary: self.boundary,
191 cooldown: match self.cooldown {
192 Some(cooldown) => Some((provider_key, cooldown)),
193 None => None,
194 },
195 }
196 }
197}
198
199pub async fn admit_host_charge_submission(
200 pool: &PgPool,
201 targets: &dyn HostChargeTargetStore,
202 reservation: &HostChargeReservation,
203) -> Result<HostChargeAdmissionOutcome, HostChargeApplicationError> {
204 let mut transaction = pool.begin().await?;
205 let outcome =
206 admit_host_charge_submission_in_transaction(&mut transaction, targets, reservation).await?;
207 transaction.commit().await?;
208 Ok(match outcome {
209 HostChargeSubmissionOutcome::Admitted(attempt) => {
210 HostChargeAdmissionOutcome::Admitted(Box::new(AdmittedHostCharge {
211 reservation: reservation.clone(),
212 attempt,
213 }))
214 }
215 HostChargeSubmissionOutcome::AlreadyAdmitted(attempt) => {
216 HostChargeAdmissionOutcome::AlreadyAdmitted(attempt)
217 }
218 HostChargeSubmissionOutcome::Rejected { attempt, reason } => {
219 HostChargeAdmissionOutcome::Rejected { attempt, reason }
220 }
221 })
222}
223
224pub async fn submit_admitted_host_charge(
225 pool: &PgPool,
226 coordinator: &dyn BillingTransactionCoordinator,
227 targets: &dyn HostChargeTargetStore,
228 admission: AdmittedHostCharge,
229 command: &ChargeHostTarget,
230 gateway: ModeVerifiedGateway<'_>,
231) -> Result<HostChargeProviderResult, HostChargeApplicationError> {
232 let resolved_gateway = gateway.resolved_gateway();
233 let reconstructed = HostChargeReservation::from_command(
234 command,
235 admission.reservation.snapshot(),
236 resolved_gateway,
237 admission.attempt.identity().attempt_id(),
238 admission
239 .reservation
240 .identity()
241 .required_gateway_account_mode(),
242 )
243 .map_err(|_| HostChargeApplicationError::SubmissionIdentityMismatch)?;
244 if reconstructed != admission.reservation
245 || admission.attempt.identity() != admission.reservation.identity()
246 || admission.attempt.request() != admission.reservation.request()
247 || admission.attempt.status() != PaymentAttemptStatus::Pending
248 || admission
249 .attempt
250 .state()
251 .timestamps()
252 .submitted_at()
253 .is_none()
254 {
255 return Err(HostChargeApplicationError::SubmissionIdentityMismatch);
256 }
257 let Some(gateway) = gateway.authorize_attempt(&admission.reservation.identity()) else {
258 return Err(HostChargeApplicationError::SubmissionIdentityMismatch);
259 };
260 let request = GatewaySaleRequest::new(
261 admission.reservation.snapshot().charge(),
262 admission.attempt.request().gateway_order_id().clone(),
263 GatewaySaleIntent::OneTime {
264 payment_token: command.payment_token().clone(),
265 },
266 command.billing_contact().cloned(),
267 );
268 let provider_key = gateway.provider_key().clone();
269 match gateway.sale(request).await {
270 Ok(outcome) => apply_host_charge_gateway_outcome(
271 pool,
272 coordinator,
273 targets,
274 &admission.reservation,
275 &outcome,
276 )
277 .await
278 .map(HostChargeProviderResult::Payment),
279 Err(GatewayMutationError::NotSubmitted(error)) => {
280 let evidence = mutation_error_evidence(error.detail());
281 let policy = GatewayNotSubmittedPolicy::for_error(&error);
282 let application = if policy.restores_prepared_attempt_when_supported() {
283 restore_admitted_host_charge_for_retry(pool, &admission.reservation).await?
284 } else {
285 resolve_host_charge_non_approved(
286 pool,
287 targets,
288 &admission.reservation,
289 &evidence,
290 AttemptResolutionStatus::Failed,
291 Some(policy.resolution_code()),
292 HostChargeBeforeSubmissionResolution::admitted_not_submitted()
293 .with_not_submitted_policy(policy)
294 .with_provider(&provider_key),
295 )
296 .await?
297 };
298 if application.should_surface_not_submitted(policy) {
299 Ok(HostChargeProviderResult::NotSubmitted {
300 payment: application.payment,
301 error,
302 })
303 } else {
304 Ok(HostChargeProviderResult::Payment(application.payment))
305 }
306 }
307 Err(GatewayMutationError::RateLimitedIndeterminate(detail)) => resolve_host_charge_unknown(
308 pool,
309 &admission.reservation,
310 &mutation_error_evidence(&detail),
311 Some((&provider_key, RateLimitCooldown::Provider)),
312 )
313 .await
314 .map(HostChargeProviderResult::Payment),
315 Err(GatewayMutationError::Indeterminate(detail)) => resolve_host_charge_unknown(
316 pool,
317 &admission.reservation,
318 &mutation_error_evidence(&detail),
319 None,
320 )
321 .await
322 .map(HostChargeProviderResult::Payment),
323 }
324}
325
326pub async fn apply_host_charge_gateway_outcome(
327 pool: &PgPool,
328 coordinator: &dyn BillingTransactionCoordinator,
329 targets: &dyn HostChargeTargetStore,
330 reservation: &HostChargeReservation,
331 outcome: &GatewayPaymentOutcome,
332) -> Result<HostChargePaymentResult, HostChargeApplicationError> {
333 apply_host_charge_gateway_decision(pool, coordinator, targets, reservation, outcome)
334 .await
335 .map(|result| result.with_gateway_diagnostics(outcome.diagnostics().to_vec()))
336}
337
338async fn apply_host_charge_gateway_decision(
339 pool: &PgPool,
340 coordinator: &dyn BillingTransactionCoordinator,
341 targets: &dyn HostChargeTargetStore,
342 reservation: &HostChargeReservation,
343 outcome: &GatewayPaymentOutcome,
344) -> Result<HostChargePaymentResult, HostChargeApplicationError> {
345 match outcome.status() {
346 GatewayPaymentStatus::Approved => {
347 let approved_evidence =
348 outcome
349 .approved_evidence()
350 .ok_or(HostChargeApplicationError::InvalidState(
351 INVALID_HOST_CHARGE_STATE,
352 ))?;
353 if outcome.transaction_id().is_none() {
354 return durably_park_host_charge_approved(pool, reservation, &approved_evidence)
355 .await;
356 }
357 for attempt_index in 0..APPROVED_APPLICATION_ATTEMPTS {
358 match apply_host_charge_approved(
359 coordinator,
360 targets,
361 reservation,
362 &approved_evidence,
363 )
364 .await
365 {
366 Ok(result) => return Ok(result),
367 Err(_) if attempt_index + 1 < APPROVED_APPLICATION_ATTEMPTS => {
368 tokio::time::sleep(APPROVED_EVIDENCE_RETRY_DELAY).await;
369 }
370 Err(_) => break,
371 }
372 }
373 durably_park_host_charge_approved(pool, reservation, &approved_evidence).await
374 }
375 GatewayPaymentStatus::Declined => resolve_host_charge_non_approved(
376 pool,
377 targets,
378 reservation,
379 outcome.evidence(),
380 AttemptResolutionStatus::Declined,
381 None,
382 HostChargeNonApprovedResolution::submitted(),
383 )
384 .await
385 .map(HostChargeResolutionApplication::into_payment),
386 GatewayPaymentStatus::Failed => resolve_host_charge_non_approved(
387 pool,
388 targets,
389 reservation,
390 outcome.evidence(),
391 AttemptResolutionStatus::Failed,
392 None,
393 HostChargeNonApprovedResolution::submitted(),
394 )
395 .await
396 .map(HostChargeResolutionApplication::into_payment),
397 GatewayPaymentStatus::Unknown => {
398 resolve_host_charge_unknown(pool, reservation, outcome.evidence(), None).await
399 }
400 }
401}
402
403pub async fn apply_reconciled_host_charge_gateway_outcome(
404 pool: &PgPool,
405 coordinator: &dyn BillingTransactionCoordinator,
406 targets: &dyn HostChargeTargetStore,
407 billing_scope_id: syrup_rail::BillingScopeId,
408 attempt_id: syrup_rail::PaymentAttemptId,
409 outcome: &GatewayPaymentOutcome,
410) -> Result<HostChargePaymentResult, HostChargeApplicationError> {
411 let mut transaction = pool.begin().await?;
412 let attempt = crate::find_payment_attempt_by_id_in_transaction(
413 &mut transaction,
414 billing_scope_id,
415 attempt_id,
416 )
417 .await?
418 .ok_or(HostChargeApplicationError::InvalidState(
419 INVALID_HOST_CHARGE_STATE,
420 ))?;
421 transaction.commit().await?;
422 let reservation = HostChargeReservation::from_attempt(&attempt)
423 .map_err(|_| HostChargeApplicationError::InvalidState(INVALID_HOST_CHARGE_STATE))?;
424 apply_host_charge_gateway_outcome(pool, coordinator, targets, &reservation, outcome).await
425}
426
427pub(crate) async fn resolve_host_charge_before_submission(
428 pool: &PgPool,
429 targets: &dyn HostChargeTargetStore,
430 reservation: &HostChargeReservation,
431 provider_key: &GatewayProviderKey,
432 detail: syrup_rail::GatewayDiagnostic,
433 resolution_code: PaymentResolutionCode,
434 resolution: HostChargeBeforeSubmissionResolution,
435) -> Result<HostChargePaymentResult, HostChargeApplicationError> {
436 let condition = if matches!(
437 resolution_code,
438 PaymentResolutionCode::GatewayAccountMutationCooldownBeforeSubmission
439 | PaymentResolutionCode::GatewayAccountRateLimitedBeforeSubmission
440 | PaymentResolutionCode::GatewayProviderRateLimitedBeforeSubmission
441 ) {
442 None
443 } else {
444 Some(syrup_rail::GatewayDiagnostic::new("failed"))
445 };
446 let evidence = ProcessorEvidence::new(
447 None,
448 None,
449 None,
450 None,
451 Some(detail),
452 condition,
453 syrup_rail::GatewayPaymentDescriptor::default(),
454 );
455 resolve_host_charge_non_approved(
456 pool,
457 targets,
458 reservation,
459 &evidence,
460 AttemptResolutionStatus::Failed,
461 Some(resolution_code),
462 resolution.with_provider(provider_key),
463 )
464 .await
465 .map(HostChargeResolutionApplication::into_payment)
466}
467
468async fn apply_host_charge_approved(
469 coordinator: &dyn BillingTransactionCoordinator,
470 targets: &dyn HostChargeTargetStore,
471 reservation: &HostChargeReservation,
472 approved_evidence: &ApprovedProcessorEvidence,
473) -> Result<HostChargePaymentResult, HostChargeApplicationError> {
474 let evidence = approved_evidence.evidence();
475 let identity = reservation.identity();
476 let mut transaction = coordinator
477 .begin(
478 BillingEventSubject::new(identity.billing_scope_id(), identity.subscriber_id()),
479 BILLING_LOCK_TIMEOUT,
480 )
481 .await?;
482 if transaction.subject_state() != BillingTransactionSubjectState::LiveRecipient {
483 let _ = transaction.rollback().await;
484 return Err(HostChargeApplicationError::InvalidState(
485 "a host charge payment event requires a live recipient",
486 ));
487 }
488 let connection = transaction.connection();
489 set_application_timeouts(connection).await?;
490 let effective_at: DateTime<Utc> = sqlx::query_scalar("SELECT clock_timestamp()")
491 .fetch_one(&mut *connection)
492 .await?;
493 let target_outcome = targets
494 .apply_transition(
495 connection,
496 HostChargeTargetTransition::new(
497 identity.billing_scope_id(),
498 identity.subscriber_id(),
499 identity.attempt_id(),
500 reservation.snapshot().target_id(),
501 HostChargeTargetTransitionKind::Paid,
502 effective_at,
503 ),
504 )
505 .await?;
506 let attempt = lock_expected_host_charge(connection, reservation).await?;
507 if attempt.status() == PaymentAttemptStatus::Approved {
508 observe_processor_charge(
513 connection,
514 &attempt,
515 evidence,
516 ProcessorChargeProgression::Applied,
517 )
518 .await?;
519 transaction.commit().await?;
520 return Ok(HostChargePaymentResult::new(attempt)?);
521 }
522 if attempt.status().is_terminal() {
523 transaction.rollback().await?;
524 return observe_terminal_host_charge_approval(
525 coordinator,
526 reservation,
527 &attempt,
528 approved_evidence,
529 )
530 .await;
531 }
532 let observation = observe_processor_charge(
533 connection,
534 &attempt,
535 evidence,
536 ProcessorChargeProgression::Pending,
537 )
538 .await?;
539 let ObservedCharge::Owned(charge) = observation else {
540 let _ = transaction.rollback().await;
541 return Err(HostChargeApplicationError::InvalidState(
542 "the approved gateway transaction belongs to another payment attempt",
543 ));
544 };
545 if charge.role == ProcessorChargeRole::Additional {
546 let _ = transaction.rollback().await;
547 return Err(HostChargeApplicationError::InvalidState(
548 "an additional approved host charge requires external reversal",
549 ));
550 }
551 match target_outcome {
552 HostChargeTargetTransitionOutcome::Applied
553 | HostChargeTargetTransitionOutcome::ExactReplay => {
554 persist_attempt_transition(
555 connection,
556 &attempt,
557 evidence,
558 AttemptTransition::Approved(AttemptApproval::HostCharge),
559 )
560 .await
561 .map_err(map_attempt_transition_error)?;
562 transition_charge(
563 connection,
564 charge.id,
565 ProcessorChargeProgression::Applied,
566 None,
567 )
568 .await?;
569 let applied = find_payment_attempt_by_id_on_connection(
570 connection,
571 identity.billing_scope_id(),
572 identity.attempt_id(),
573 )
574 .await?
575 .ok_or(HostChargeApplicationError::InvalidState(
576 INVALID_HOST_CHARGE_STATE,
577 ))?;
578 let event = BillingEvent::HostChargePaid {
579 attempt_id: identity.attempt_id(),
580 target_id: reservation.snapshot().target_id(),
581 charge: reservation.snapshot().charge(),
582 };
583 transaction.append_event(&event).await?;
584 transaction.commit().await?;
585 Ok(HostChargePaymentResult::new(applied)?)
586 }
587 HostChargeTargetTransitionOutcome::StaleTarget => {
588 transition_charge(
589 connection,
590 charge.id,
591 ProcessorChargeProgression::ExternalReversalRequired,
592 Some(PaymentResolutionCode::HostChargeApprovedStaleState),
593 )
594 .await?;
595 let parked = park_locked_attempt(
596 connection,
597 &attempt,
598 evidence,
599 Some(PaymentResolutionCode::HostChargeApprovedStaleState),
600 APPROVED_STALE_TARGET_TEXT,
601 )
602 .await?;
603 transaction.commit().await?;
604 Ok(HostChargePaymentResult::new(parked)?)
605 }
606 HostChargeTargetTransitionOutcome::Unchanged { .. } => {
607 let _ = transaction.rollback().await;
608 Err(HostChargeApplicationError::InvalidState(
609 INVALID_HOST_CHARGE_STATE,
610 ))
611 }
612 }
613}
614
615async fn observe_terminal_host_charge_approval(
616 coordinator: &dyn BillingTransactionCoordinator,
617 reservation: &HostChargeReservation,
618 terminal_attempt: &PaymentAttempt,
619 approved_evidence: &ApprovedProcessorEvidence,
620) -> Result<HostChargePaymentResult, HostChargeApplicationError> {
621 let evidence = approved_evidence.evidence();
622 let identity = reservation.identity();
623 let mut transaction = coordinator
624 .begin(
625 BillingEventSubject::new(identity.billing_scope_id(), identity.subscriber_id()),
626 BILLING_LOCK_TIMEOUT,
627 )
628 .await?;
629 let connection = transaction.connection();
630 set_application_timeouts(connection).await?;
631 if matches!(
632 observe_processor_charge(
633 connection,
634 terminal_attempt,
635 evidence,
636 ProcessorChargeProgression::Pending,
637 )
638 .await?,
639 ObservedCharge::OwnedByOtherAttempt
640 ) {
641 let _ = transaction.rollback().await;
642 return Err(HostChargeApplicationError::InvalidState(
643 "the approved gateway transaction belongs to another payment attempt",
644 ));
645 }
646 let locked = lock_expected_host_charge(connection, reservation).await?;
647 if !locked.status().is_terminal() || locked.status() == PaymentAttemptStatus::Approved {
648 let _ = transaction.rollback().await;
649 return Err(HostChargeApplicationError::InvalidState(
650 INVALID_HOST_CHARGE_STATE,
651 ));
652 }
653 transaction.commit().await?;
654 Ok(HostChargePaymentResult::confirmation_pending(
655 locked,
656 approved_evidence.clone(),
657 )?)
658}
659
660async fn resolve_host_charge_non_approved(
661 pool: &PgPool,
662 targets: &dyn HostChargeTargetStore,
663 reservation: &HostChargeReservation,
664 evidence: &ProcessorEvidence,
665 status: AttemptResolutionStatus,
666 resolution_code: Option<PaymentResolutionCode>,
667 resolution: HostChargeNonApprovedResolution<'_>,
668) -> Result<HostChargeResolutionApplication, HostChargeApplicationError> {
669 let identity = reservation.identity();
670 if let Some((provider_key, cooldown)) = resolution.cooldown {
677 commit_host_charge_cooldown(pool, reservation, provider_key, cooldown).await?;
678 }
679 let mut transaction = pool.begin().await?;
680 set_application_timeouts(&mut transaction).await?;
681 let effective_at = sqlx::query_scalar("SELECT clock_timestamp()")
682 .fetch_one(&mut *transaction)
683 .await?;
684 let target_outcome = targets
685 .apply_transition(
686 &mut transaction,
687 HostChargeTargetTransition::new(
688 identity.billing_scope_id(),
689 identity.subscriber_id(),
690 identity.attempt_id(),
691 reservation.snapshot().target_id(),
692 match resolution.boundary {
693 OutcomeResolutionBoundary::Prepared
694 | OutcomeResolutionBoundary::AdmittedNotSubmitted => {
695 HostChargeTargetTransitionKind::ReleasedBeforeSubmission
696 }
697 OutcomeResolutionBoundary::Submitted => {
698 HostChargeTargetTransitionKind::PaymentFailed
699 }
700 },
701 effective_at,
702 ),
703 )
704 .await?;
705 if !target_outcome.is_applied() {
706 tracing::warn!(
707 target: "syrup_rail::host_charge_target",
708 billing_scope_id = %identity.billing_scope_id().as_uuid(),
709 subscriber_id = %identity.subscriber_id().as_uuid(),
710 attempt_id = %identity.attempt_id().as_uuid(),
711 target_id = %reservation.snapshot().target_id().as_uuid(),
712 boundary = ?resolution.boundary,
713 ?target_outcome,
714 "host target refused a payment outcome; leaving the canonical attempt unresolved"
715 );
716 transaction.rollback().await?;
717 let application = canonical_host_charge_resolution_application(pool, reservation).await?;
718 if !host_charge_attempt_may_resolve(application.payment.attempt(), resolution.boundary) {
719 return Ok(application);
720 }
721 return Err(HostChargeApplicationError::InvalidState(
722 INVALID_HOST_CHARGE_STATE,
723 ));
724 }
725 let attempt = lock_expected_host_charge(&mut transaction, reservation).await?;
726 let may_resolve = host_charge_attempt_may_resolve(&attempt, resolution.boundary);
727 if !may_resolve {
728 transaction.rollback().await?;
729 return canonical_host_charge_resolution_application(pool, reservation).await;
730 }
731 persist_attempt_transition(
732 &mut transaction,
733 &attempt,
734 evidence,
735 AttemptTransition::Resolved {
736 status,
737 resolution_code,
738 },
739 )
740 .await
741 .map_err(map_attempt_transition_error)?;
742 if evidence.indicates_approved_payment() {
743 observe_processor_charge(
744 &mut transaction,
745 &attempt,
746 evidence,
747 ProcessorChargeProgression::Pending,
748 )
749 .await?;
750 }
751 if resolution.boundary == OutcomeResolutionBoundary::AdmittedNotSubmitted {
752 let cleared = sqlx::query(
753 "UPDATE billing_payment_attempts SET submitted_at = NULL, updated_at = clock_timestamp() WHERE id = $1 AND status = $2",
754 )
755 .bind(identity.attempt_id().as_uuid())
756 .bind(status.as_str())
757 .execute(&mut *transaction)
758 .await?;
759 if cleared.rows_affected() != 1 {
760 return Err(HostChargeApplicationError::InvalidState(
761 INVALID_HOST_CHARGE_STATE,
762 ));
763 }
764 }
765 let attempt = find_payment_attempt_by_id_on_connection(
766 &mut transaction,
767 identity.billing_scope_id(),
768 identity.attempt_id(),
769 )
770 .await?
771 .ok_or(HostChargeApplicationError::InvalidState(
772 INVALID_HOST_CHARGE_STATE,
773 ))?;
774 transaction.commit().await?;
775 Ok(HostChargeResolutionApplication {
776 payment: HostChargePaymentResult::new(attempt)?,
777 applied: true,
778 })
779}
780
781fn host_charge_attempt_may_resolve(
782 attempt: &PaymentAttempt,
783 boundary: OutcomeResolutionBoundary,
784) -> bool {
785 attempt.status().is_resolvable()
786 && match boundary {
787 OutcomeResolutionBoundary::Prepared => {
788 attempt.state().timestamps().submitted_at().is_none()
789 }
790 OutcomeResolutionBoundary::AdmittedNotSubmitted => {
791 attempt.state().timestamps().submitted_at().is_some()
792 }
793 OutcomeResolutionBoundary::Submitted => true,
794 }
795}
796
797async fn canonical_host_charge_resolution_application(
798 pool: &PgPool,
799 reservation: &HostChargeReservation,
800) -> Result<HostChargeResolutionApplication, HostChargeApplicationError> {
801 let identity = reservation.identity();
802 let mut reload = pool.begin().await?;
803 let attempt = crate::find_payment_attempt_by_id_in_transaction(
804 &mut reload,
805 identity.billing_scope_id(),
806 identity.attempt_id(),
807 )
808 .await?
809 .ok_or(HostChargeApplicationError::InvalidState(
810 INVALID_HOST_CHARGE_STATE,
811 ))?;
812 reload.commit().await?;
813 Ok(HostChargeResolutionApplication {
814 payment: HostChargePaymentResult::new(attempt)?,
815 applied: false,
816 })
817}
818
819async fn restore_admitted_host_charge_for_retry(
820 pool: &PgPool,
821 reservation: &HostChargeReservation,
822) -> Result<HostChargeResolutionApplication, HostChargeApplicationError> {
823 let identity = reservation.identity();
827 let mut transaction = pool.begin().await?;
828 set_application_timeouts(&mut transaction).await?;
829 let attempt = lock_expected_host_charge(&mut transaction, reservation).await?;
830 let restored = attempt.status() == PaymentAttemptStatus::Pending
831 && attempt.state().timestamps().submitted_at().is_some();
832 if restored {
833 restore_prepared_attempt_submission(&mut transaction, &attempt).await?;
834 }
835 let attempt = find_payment_attempt_by_id_on_connection(
836 &mut transaction,
837 identity.billing_scope_id(),
838 identity.attempt_id(),
839 )
840 .await?
841 .ok_or(HostChargeApplicationError::InvalidState(
842 INVALID_HOST_CHARGE_STATE,
843 ))?;
844 transaction.commit().await?;
845 if restored {
846 tracing::warn!(
847 target: "syrup_rail::gateway_control_plane",
848 attempt_id = %identity.attempt_id().as_uuid(),
849 attempt_kind = attempt.kind().as_str(),
850 required_gateway_account_mode = identity.required_gateway_account_mode().as_str(),
851 "restored admitted host charge after pre-submission control-plane failure"
852 );
853 }
854 Ok(HostChargeResolutionApplication {
855 payment: HostChargePaymentResult::new(attempt)?,
856 applied: restored,
857 })
858}
859
860async fn resolve_host_charge_unknown(
861 pool: &PgPool,
862 reservation: &HostChargeReservation,
863 evidence: &ProcessorEvidence,
864 cooldown: Option<(&GatewayProviderKey, RateLimitCooldown)>,
865) -> Result<HostChargePaymentResult, HostChargeApplicationError> {
866 let identity = reservation.identity();
867 if let Some((provider_key, cooldown)) = cooldown {
868 commit_host_charge_cooldown(pool, reservation, provider_key, cooldown).await?;
869 }
870 let mut transaction = pool.begin().await?;
871 set_application_timeouts(&mut transaction).await?;
872 let attempt = lock_expected_host_charge(&mut transaction, reservation).await?;
873 if !attempt.status().is_terminal() {
874 persist_attempt_transition(
875 &mut transaction,
876 &attempt,
877 evidence,
878 AttemptTransition::Resolved {
879 status: AttemptResolutionStatus::Unknown,
880 resolution_code: None,
881 },
882 )
883 .await
884 .map_err(map_attempt_transition_error)?;
885 if evidence.indicates_approved_payment() {
886 observe_processor_charge(
887 &mut transaction,
888 &attempt,
889 evidence,
890 ProcessorChargeProgression::Pending,
891 )
892 .await?;
893 }
894 }
895 let attempt = find_payment_attempt_by_id_on_connection(
896 &mut transaction,
897 identity.billing_scope_id(),
898 identity.attempt_id(),
899 )
900 .await?
901 .ok_or(HostChargeApplicationError::InvalidState(
902 INVALID_HOST_CHARGE_STATE,
903 ))?;
904 transaction.commit().await?;
905 Ok(HostChargePaymentResult::new(attempt)?)
906}
907
908async fn commit_host_charge_cooldown(
909 pool: &PgPool,
910 reservation: &HostChargeReservation,
911 provider_key: &GatewayProviderKey,
912 cooldown: RateLimitCooldown,
913) -> Result<(), HostChargeApplicationError> {
914 match commit_rate_limit_cooldown_for_operation(
915 pool,
916 reservation.identity(),
917 provider_key,
918 cooldown,
919 RateLimitCooldownOperation::HostCharge,
920 )
921 .await
922 {
923 Ok(()) => Ok(()),
924 Err(RateLimitCooldownCommitError::Sql(error)) => Err(error.into()),
925 Err(RateLimitCooldownCommitError::MissingProviderCooldown) => Err(
926 HostChargeApplicationError::InvalidState(INVALID_HOST_CHARGE_STATE),
927 ),
928 }
929}
930
931async fn park_host_charge_approved(
932 pool: &PgPool,
933 reservation: &HostChargeReservation,
934 evidence: &ProcessorEvidence,
935) -> Result<HostChargePaymentResult, HostChargeApplicationError> {
936 let mut transaction = pool.begin().await?;
937 set_application_timeouts(&mut transaction).await?;
938 let attempt = lock_expected_host_charge(&mut transaction, reservation).await?;
939 let progression = if evidence.transaction_id().is_some() {
940 ProcessorChargeProgression::ExternalReversalRequired
941 } else {
942 ProcessorChargeProgression::ReconciliationRequired
943 };
944 let observation =
945 observe_processor_charge(&mut transaction, &attempt, evidence, progression).await?;
946 if let ObservedCharge::Owned(charge) = observation {
947 transition_charge(&mut transaction, charge.id, progression, None).await?;
948 }
949 let parked = park_locked_attempt(
950 &mut transaction,
951 &attempt,
952 evidence,
953 None,
954 APPROVED_STORAGE_FAILURE_TEXT,
955 )
956 .await?;
957 transaction.commit().await?;
958 Ok(HostChargePaymentResult::new(parked)?)
959}
960
961async fn durably_park_host_charge_approved(
962 pool: &PgPool,
963 reservation: &HostChargeReservation,
964 approved_evidence: &ApprovedProcessorEvidence,
965) -> Result<HostChargePaymentResult, HostChargeApplicationError> {
966 let evidence = approved_evidence.evidence();
967 if let Ok(payment) = park_host_charge_approved(pool, reservation, evidence).await {
968 return Ok(payment);
969 }
970 crate::store_compensating_processor_charge(
971 pool,
972 reservation.identity().attempt_id(),
973 reservation.request().gateway_order_id(),
974 evidence,
975 )
976 .await?;
977 let mut transaction = pool.begin().await?;
978 let attempt = crate::find_payment_attempt_by_id_in_transaction(
979 &mut transaction,
980 reservation.identity().billing_scope_id(),
981 reservation.identity().attempt_id(),
982 )
983 .await?
984 .ok_or(HostChargeApplicationError::InvalidState(
985 INVALID_HOST_CHARGE_STATE,
986 ))?;
987 transaction.commit().await?;
988 Ok(HostChargePaymentResult::confirmation_pending(
989 attempt,
990 approved_evidence.clone(),
991 )?)
992}
993
994async fn lock_expected_host_charge(
995 connection: &mut PgConnection,
996 reservation: &HostChargeReservation,
997) -> Result<PaymentAttempt, HostChargeApplicationError> {
998 let identity = reservation.identity();
999 let attempt = lock_payment_attempt_by_id_on_connection(
1000 connection,
1001 identity.billing_scope_id(),
1002 identity.attempt_id(),
1003 )
1004 .await?
1005 .ok_or(HostChargeApplicationError::InvalidState(
1006 INVALID_HOST_CHARGE_STATE,
1007 ))?;
1008 if !host_charge_attempt_matches_reservation(&attempt, reservation) {
1009 return Err(HostChargeApplicationError::InvalidState(
1010 INVALID_HOST_CHARGE_STATE,
1011 ));
1012 }
1013 Ok(attempt)
1014}
1015
1016#[cfg(test)]
1017mod tests {
1018 use std::{
1019 error::Error,
1020 sync::{
1021 Arc,
1022 atomic::{AtomicUsize, Ordering},
1023 },
1024 };
1025
1026 use async_trait::async_trait;
1027 use chrono::Duration as ChronoDuration;
1028 use sqlx::{Postgres, Transaction};
1029 use syrup_rail::{
1030 BillingContact, BillingEventKey, ChargeAmount, CurrencyCode, EndUserMutationAdmission,
1031 EndUserMutationAdmissionResult, EndUserMutationCommand, GatewayAccountId,
1032 GatewayAccountMode, GatewayConfigurationId, GatewayDiagnostic, GatewayError,
1033 GatewayLifecycleCursorKey, GatewayLifecycleQueryPolicy, GatewayMutationError,
1034 GatewayMutationReferenceFactory, GatewayOrderId, GatewayPaymentDescriptor,
1035 GatewayPaymentDiagnostic, GatewayProviderKey, GatewayQueryRequest, GatewayResolutionError,
1036 GatewayResolver, GatewayStorePaymentMethodRequest, GatewayTransactionId,
1037 GatewayTransactionReport, GatewayTransactionReportRequest, HostChargeTargetId,
1038 HostChargeTargetNoChange, IdempotencyKey, PaymentAttemptId, PaymentAttemptKind,
1039 PaymentGateway, PaymentToken, ResolvedGateway,
1040 };
1041 use tokio::sync::{Mutex, Notify, oneshot};
1042 use uuid::Uuid;
1043
1044 use super::*;
1045 use crate::{
1046 BillingEventWriteError, BillingTransaction, HostChargeLedgerAdmission,
1047 HostChargeLedgerAdmissionMode, HostChargeLedgerAdmissionQuery,
1048 HostChargeReservationDecision, HostChargeReservationOutcome, HostChargeSubmissionAdmission,
1049 HostChargeSubmissionDecision, HostChargeTargetReservation, SubscriptionBillingService,
1050 SubscriptionOfferStore, host_charge_ledger_admission, reserve_host_charge_in_transaction,
1051 test_support::{TestDatabase, create_gateway_account},
1052 };
1053
1054 mod readiness_replay;
1055
1056 #[test]
1057 fn before_submission_resolution_modes_keep_boundary_and_cooldown_distinct() {
1058 let prepared = HostChargeBeforeSubmissionResolution::prepared();
1059 assert_eq!(prepared.boundary, OutcomeResolutionBoundary::Prepared);
1060 assert!(prepared.cooldown.is_none());
1061
1062 let admitted = HostChargeBeforeSubmissionResolution::admitted_not_submitted();
1063 assert_eq!(
1064 admitted.boundary,
1065 OutcomeResolutionBoundary::AdmittedNotSubmitted
1066 );
1067 assert!(admitted.cooldown.is_none());
1068
1069 let rate_limited = HostChargeBeforeSubmissionResolution::prepared()
1070 .with_not_submitted_policy(GatewayNotSubmittedPolicy::for_readiness_error(
1071 &syrup_rail::GatewayError::RateLimited(GatewayDiagnostic::new("rate limited")),
1072 ));
1073 assert_eq!(rate_limited.boundary, OutcomeResolutionBoundary::Prepared);
1074 assert!(matches!(
1075 rate_limited.cooldown,
1076 Some(RateLimitCooldown::Provider)
1077 ));
1078
1079 let terminal_failure = HostChargeBeforeSubmissionResolution::prepared()
1080 .with_not_submitted_policy(GatewayNotSubmittedPolicy::for_readiness_error(
1081 &syrup_rail::GatewayError::Configuration(GatewayDiagnostic::new("configuration")),
1082 ));
1083 assert_eq!(
1084 terminal_failure.boundary,
1085 OutcomeResolutionBoundary::Prepared
1086 );
1087 assert!(terminal_failure.cooldown.is_none());
1088 }
1089
1090 struct TestReferenceFactory;
1091
1092 impl GatewayMutationReferenceFactory for TestReferenceFactory {
1093 fn for_attempt(
1094 &self,
1095 _kind: PaymentAttemptKind,
1096 attempt_id: PaymentAttemptId,
1097 ) -> GatewayOrderId {
1098 GatewayOrderId::from_generated_attempt(
1099 format!("test_host_{}", attempt_id.as_uuid().simple()),
1100 attempt_id,
1101 )
1102 .expect("valid host test reference")
1103 }
1104 }
1105
1106 struct ScriptedGateway {
1107 account_mode: GatewayAccountMode,
1108 sale_calls: AtomicUsize,
1109 outcome: Mutex<Option<GatewayPaymentOutcome>>,
1110 }
1111
1112 #[async_trait]
1113 impl PaymentGateway for ScriptedGateway {
1114 async fn account_mode(&self) -> Result<GatewayAccountMode, GatewayError> {
1115 Ok(self.account_mode)
1116 }
1117
1118 async fn sale(
1119 &self,
1120 _request: GatewaySaleRequest,
1121 ) -> Result<GatewayPaymentOutcome, GatewayMutationError> {
1122 self.sale_calls.fetch_add(1, Ordering::SeqCst);
1123 Ok(self
1124 .outcome
1125 .lock()
1126 .await
1127 .take()
1128 .expect("one sale capability"))
1129 }
1130
1131 async fn store_payment_method(
1132 &self,
1133 _request: GatewayStorePaymentMethodRequest,
1134 ) -> Result<GatewayPaymentOutcome, GatewayMutationError> {
1135 panic!("host charge must not store a payment method")
1136 }
1137
1138 async fn query_transaction(
1139 &self,
1140 _request: GatewayQueryRequest,
1141 ) -> Result<Option<GatewayPaymentOutcome>, GatewayError> {
1142 panic!("foreground host charge must not query")
1143 }
1144
1145 async fn query_transaction_reports(
1146 &self,
1147 _request: GatewayTransactionReportRequest,
1148 ) -> Result<Vec<GatewayTransactionReport>, GatewayError> {
1149 panic!("foreground host charge must not query reports")
1150 }
1151 }
1152
1153 struct RateLimitedAfterReservationGateway {
1154 readiness_calls: AtomicUsize,
1155 sale_calls: AtomicUsize,
1156 }
1157
1158 struct TerminalRaceGateway {
1159 pool: PgPool,
1160 sale_calls: AtomicUsize,
1161 }
1162
1163 struct RacingPreparedRetryGateway {
1164 readiness_calls: AtomicUsize,
1165 sale_calls: AtomicUsize,
1166 blocked_readiness_started: Mutex<Option<oneshot::Sender<()>>>,
1167 sale_started: Mutex<Option<oneshot::Sender<()>>>,
1168 release_readiness: Notify,
1169 release_sale: Notify,
1170 }
1171
1172 #[async_trait]
1173 impl PaymentGateway for RacingPreparedRetryGateway {
1174 async fn account_mode(&self) -> Result<GatewayAccountMode, GatewayError> {
1175 if self.readiness_calls.fetch_add(1, Ordering::SeqCst) == 0 {
1176 if let Some(started) = self.blocked_readiness_started.lock().await.take() {
1177 let _ = started.send(());
1178 }
1179 self.release_readiness.notified().await;
1180 Ok(GatewayAccountMode::Test)
1181 } else {
1182 Ok(GatewayAccountMode::Live)
1183 }
1184 }
1185
1186 async fn sale(
1187 &self,
1188 _request: GatewaySaleRequest,
1189 ) -> Result<GatewayPaymentOutcome, GatewayMutationError> {
1190 self.sale_calls.fetch_add(1, Ordering::SeqCst);
1191 if let Some(started) = self.sale_started.lock().await.take() {
1192 let _ = started.send(());
1193 }
1194 self.release_sale.notified().await;
1195 Ok(approved_outcome("host_txn_prepared_retry"))
1196 }
1197
1198 async fn store_payment_method(
1199 &self,
1200 _request: GatewayStorePaymentMethodRequest,
1201 ) -> Result<GatewayPaymentOutcome, GatewayMutationError> {
1202 panic!("host charge must not store a payment method")
1203 }
1204
1205 async fn query_transaction(
1206 &self,
1207 _request: GatewayQueryRequest,
1208 ) -> Result<Option<GatewayPaymentOutcome>, GatewayError> {
1209 panic!("foreground host charge must not query")
1210 }
1211
1212 async fn query_transaction_reports(
1213 &self,
1214 _request: GatewayTransactionReportRequest,
1215 ) -> Result<Vec<GatewayTransactionReport>, GatewayError> {
1216 panic!("foreground host charge must not query reports")
1217 }
1218 }
1219
1220 #[async_trait]
1221 impl PaymentGateway for TerminalRaceGateway {
1222 async fn account_mode(&self) -> Result<GatewayAccountMode, GatewayError> {
1223 Ok(GatewayAccountMode::Live)
1224 }
1225
1226 async fn sale(
1227 &self,
1228 request: GatewaySaleRequest,
1229 ) -> Result<GatewayPaymentOutcome, GatewayMutationError> {
1230 self.sale_calls.fetch_add(1, Ordering::SeqCst);
1231 let updated = sqlx::query(
1232 r#"
1233 UPDATE billing_payment_attempts
1234 SET status = 'failed', resolved_at = clock_timestamp(),
1235 gateway_response_text = 'simulated terminal race',
1236 gateway_condition = 'failed', updated_at = clock_timestamp()
1237 WHERE gateway_order_id = $1 AND status = 'pending'
1238 "#,
1239 )
1240 .bind(request.order_id().expose())
1241 .execute(&self.pool)
1242 .await
1243 .expect("simulate a terminal attempt race");
1244 assert_eq!(updated.rows_affected(), 1);
1245 Ok(approved_outcome("host_txn_terminal_race"))
1246 }
1247
1248 async fn store_payment_method(
1249 &self,
1250 _request: GatewayStorePaymentMethodRequest,
1251 ) -> Result<GatewayPaymentOutcome, GatewayMutationError> {
1252 panic!("host charge must not store a payment method")
1253 }
1254
1255 async fn query_transaction(
1256 &self,
1257 _request: GatewayQueryRequest,
1258 ) -> Result<Option<GatewayPaymentOutcome>, GatewayError> {
1259 panic!("foreground host charge must not query")
1260 }
1261
1262 async fn query_transaction_reports(
1263 &self,
1264 _request: GatewayTransactionReportRequest,
1265 ) -> Result<Vec<GatewayTransactionReport>, GatewayError> {
1266 panic!("foreground host charge must not query reports")
1267 }
1268 }
1269
1270 #[async_trait]
1271 impl PaymentGateway for RateLimitedAfterReservationGateway {
1272 async fn account_mode(&self) -> Result<GatewayAccountMode, GatewayError> {
1273 self.readiness_calls.fetch_add(1, Ordering::SeqCst);
1274 Err(GatewayError::RateLimited(GatewayDiagnostic::new(
1275 "provider throttled the readiness check",
1276 )))
1277 }
1278
1279 async fn sale(
1280 &self,
1281 _request: GatewaySaleRequest,
1282 ) -> Result<GatewayPaymentOutcome, GatewayMutationError> {
1283 self.sale_calls.fetch_add(1, Ordering::SeqCst);
1284 panic!("rate-limited host charge must not submit a sale")
1285 }
1286
1287 async fn store_payment_method(
1288 &self,
1289 _request: GatewayStorePaymentMethodRequest,
1290 ) -> Result<GatewayPaymentOutcome, GatewayMutationError> {
1291 panic!("host charge must not store a payment method")
1292 }
1293
1294 async fn query_transaction(
1295 &self,
1296 _request: GatewayQueryRequest,
1297 ) -> Result<Option<GatewayPaymentOutcome>, GatewayError> {
1298 panic!("foreground host charge must not query")
1299 }
1300
1301 async fn query_transaction_reports(
1302 &self,
1303 _request: GatewayTransactionReportRequest,
1304 ) -> Result<Vec<GatewayTransactionReport>, GatewayError> {
1305 panic!("foreground host charge must not query reports")
1306 }
1307 }
1308
1309 struct StaticResolver {
1310 gateway: ResolvedGateway,
1311 calls: AtomicUsize,
1312 }
1313
1314 #[async_trait]
1315 impl GatewayResolver for StaticResolver {
1316 async fn resolve(
1317 &self,
1318 billing_scope_id: syrup_rail::BillingScopeId,
1319 gateway_account_id: GatewayAccountId,
1320 gateway_configuration_id: GatewayConfigurationId,
1321 provider_key: GatewayProviderKey,
1322 ) -> Result<ResolvedGateway, GatewayResolutionError> {
1323 self.calls.fetch_add(1, Ordering::SeqCst);
1324 if billing_scope_id != self.gateway.billing_scope_id()
1325 || gateway_account_id != self.gateway.gateway_account_id()
1326 || gateway_configuration_id != self.gateway.gateway_configuration_id()
1327 || provider_key != *self.gateway.provider_key()
1328 {
1329 return Err(GatewayResolutionError::ConfigurationChanged);
1330 }
1331 Ok(self.gateway.clone())
1332 }
1333 }
1334
1335 struct PermitAdmission {
1336 calls: AtomicUsize,
1337 }
1338
1339 #[async_trait]
1340 impl EndUserMutationAdmission for PermitAdmission {
1341 async fn admit(&self, _command: EndUserMutationCommand) -> EndUserMutationAdmissionResult {
1342 self.calls.fetch_add(1, Ordering::SeqCst);
1343 EndUserMutationAdmissionResult::Allowed
1344 }
1345 }
1346
1347 struct UnusedOffers;
1348
1349 #[async_trait]
1350 impl SubscriptionOfferStore for UnusedOffers {
1351 async fn lock_current_offer(
1352 &self,
1353 _connection: &mut PgConnection,
1354 _billing_scope_id: syrup_rail::BillingScopeId,
1355 _plan_key: &syrup_rail::PlanKey,
1356 ) -> Result<Option<syrup_rail::SubscriptionOffer>, sqlx::Error> {
1357 panic!("host charge must not load a subscription offer")
1358 }
1359 }
1360
1361 #[derive(Clone)]
1362 struct TestCoordinator {
1363 pool: PgPool,
1364 events: Arc<Mutex<Vec<BillingEvent>>>,
1365 }
1366
1367 #[async_trait]
1368 impl BillingTransactionCoordinator for TestCoordinator {
1369 async fn begin(
1370 &self,
1371 _subject: BillingEventSubject,
1372 _lock_timeout: Duration,
1373 ) -> Result<Box<dyn BillingTransaction>, BillingTransactionError> {
1374 Ok(Box::new(TestTransaction {
1375 transaction: Some(
1376 self.pool
1377 .begin()
1378 .await
1379 .map_err(BillingTransactionError::new)?,
1380 ),
1381 events: Arc::clone(&self.events),
1382 }))
1383 }
1384 }
1385
1386 struct TestTransaction {
1387 transaction: Option<Transaction<'static, Postgres>>,
1388 events: Arc<Mutex<Vec<BillingEvent>>>,
1389 }
1390
1391 #[async_trait]
1392 impl BillingTransaction for TestTransaction {
1393 fn connection(&mut self) -> &mut PgConnection {
1394 &mut *self.transaction.as_mut().expect("active transaction")
1395 }
1396
1397 fn subject_state(&self) -> BillingTransactionSubjectState {
1398 BillingTransactionSubjectState::LiveRecipient
1399 }
1400
1401 async fn append_event(
1402 &mut self,
1403 event: &BillingEvent,
1404 ) -> Result<(), BillingEventWriteError> {
1405 self.events.lock().await.push(event.clone());
1406 Ok(())
1407 }
1408
1409 async fn commit(mut self: Box<Self>) -> Result<(), BillingTransactionError> {
1410 self.transaction
1411 .take()
1412 .expect("active transaction")
1413 .commit()
1414 .await
1415 .map_err(BillingTransactionError::new)
1416 }
1417
1418 async fn rollback(mut self: Box<Self>) -> Result<(), BillingTransactionError> {
1419 self.transaction
1420 .take()
1421 .expect("active transaction")
1422 .rollback()
1423 .await
1424 .map_err(BillingTransactionError::new)
1425 }
1426 }
1427
1428 struct TestTargets;
1429
1430 struct AdmissionMustNotRun;
1431
1432 #[async_trait]
1433 impl HostChargeTargetStore for AdmissionMustNotRun {
1434 async fn preflight_target(
1435 &self,
1436 _connection: &mut PgConnection,
1437 _reservation: &HostChargeTargetReservation,
1438 ) -> Result<HostChargeReservationDecision, HostChargeTargetError> {
1439 panic!("terminal admission replay must not preflight the host target")
1440 }
1441
1442 async fn reserve_target(
1443 &self,
1444 _connection: &mut PgConnection,
1445 _reservation: &HostChargeTargetReservation,
1446 ) -> Result<HostChargeReservationDecision, HostChargeTargetError> {
1447 panic!("terminal admission replay must not reserve the host target")
1448 }
1449
1450 async fn ensure_submission_admitted(
1451 &self,
1452 _connection: &mut PgConnection,
1453 _admission: &HostChargeSubmissionAdmission,
1454 ) -> Result<HostChargeSubmissionDecision, HostChargeTargetError> {
1455 panic!("terminal admission replay must not invoke the host target")
1456 }
1457
1458 async fn apply_transition(
1459 &self,
1460 _connection: &mut PgConnection,
1461 _transition: HostChargeTargetTransition,
1462 ) -> Result<HostChargeTargetTransitionOutcome, HostChargeTargetError> {
1463 panic!("terminal admission replay must not transition the host target")
1464 }
1465 }
1466
1467 #[async_trait]
1468 impl HostChargeTargetStore for TestTargets {
1469 async fn preflight_target(
1470 &self,
1471 connection: &mut PgConnection,
1472 reservation: &HostChargeTargetReservation,
1473 ) -> Result<HostChargeReservationDecision, HostChargeTargetError> {
1474 self.reserve_target(connection, reservation).await
1475 }
1476
1477 async fn reserve_target(
1478 &self,
1479 connection: &mut PgConnection,
1480 reservation: &HostChargeTargetReservation,
1481 ) -> Result<HostChargeReservationDecision, HostChargeTargetError> {
1482 let row = sqlx::query_as::<_, (String, i32, String)>(
1483 r#"
1484 SELECT status, amount_cents, currency
1485 FROM host_charge_targets
1486 WHERE id = $1 AND billing_scope_id = $2 AND subscriber_id = $3
1487 FOR UPDATE
1488 "#,
1489 )
1490 .bind(reservation.target_id().as_uuid())
1491 .bind(reservation.billing_scope_id().as_uuid())
1492 .bind(reservation.subscriber_id().as_uuid())
1493 .fetch_optional(&mut *connection)
1494 .await
1495 .map_err(HostChargeTargetError::new)?;
1496 let Some((status, cents, currency)) = row else {
1497 return Ok(HostChargeReservationDecision::Rejected {
1498 reason: syrup_rail::HostChargeTargetRejection::TargetUnavailable,
1499 });
1500 };
1501 let ledger = host_charge_ledger_admission(
1502 connection,
1503 &HostChargeLedgerAdmissionQuery::new(
1504 reservation.billing_scope_id(),
1505 reservation.subscriber_id(),
1506 reservation.target_id(),
1507 HostChargeLedgerAdmissionMode::Reserve {
1508 idempotency_key: reservation.idempotency_key().clone(),
1509 },
1510 ),
1511 )
1512 .await
1513 .map_err(HostChargeTargetError::new)?;
1514 let charge = ChargeAmount::new(cents, CurrencyCode::new(¤cy).unwrap()).unwrap();
1515 if ledger == HostChargeLedgerAdmission::IdempotentContender {
1516 return Ok(HostChargeReservationDecision::IdempotentContender(
1517 syrup_rail::HostChargeTargetSnapshot::new(reservation.target_id(), charge),
1518 ));
1519 }
1520 if ledger != HostChargeLedgerAdmission::Safe || status != "pending" {
1521 return Ok(HostChargeReservationDecision::Rejected {
1522 reason: syrup_rail::HostChargeTargetRejection::LedgerUnsafe,
1523 });
1524 }
1525 Ok(HostChargeReservationDecision::Reserved(
1526 syrup_rail::HostChargeTargetSnapshot::new(reservation.target_id(), charge),
1527 ))
1528 }
1529
1530 async fn ensure_submission_admitted(
1531 &self,
1532 connection: &mut PgConnection,
1533 admission: &HostChargeSubmissionAdmission,
1534 ) -> Result<HostChargeSubmissionDecision, HostChargeTargetError> {
1535 let row = sqlx::query_as::<_, (String, i32, String)>(
1536 r#"
1537 SELECT status, amount_cents, currency
1538 FROM host_charge_targets
1539 WHERE id = $1 AND billing_scope_id = $2 AND subscriber_id = $3
1540 FOR UPDATE
1541 "#,
1542 )
1543 .bind(admission.target_id().as_uuid())
1544 .bind(admission.billing_scope_id().as_uuid())
1545 .bind(admission.subscriber_id().as_uuid())
1546 .fetch_optional(&mut *connection)
1547 .await
1548 .map_err(HostChargeTargetError::new)?;
1549 let Some((status, cents, currency)) = row else {
1550 return Ok(HostChargeSubmissionDecision::Rejected {
1551 reason: syrup_rail::HostChargeTargetRejection::TargetUnavailable,
1552 });
1553 };
1554 let charge = ChargeAmount::new(cents, CurrencyCode::new(¤cy).unwrap()).unwrap();
1555 let ledger = host_charge_ledger_admission(
1556 connection,
1557 &HostChargeLedgerAdmissionQuery::new(
1558 admission.billing_scope_id(),
1559 admission.subscriber_id(),
1560 admission.target_id(),
1561 HostChargeLedgerAdmissionMode::Submit {
1562 attempt_id: admission.attempt_id(),
1563 },
1564 ),
1565 )
1566 .await
1567 .map_err(HostChargeTargetError::new)?;
1568 if ledger == HostChargeLedgerAdmission::Safe
1569 && status == "pending"
1570 && charge == admission.expected_charge()
1571 {
1572 Ok(HostChargeSubmissionDecision::Admitted(
1573 syrup_rail::HostChargeTargetSnapshot::new(admission.target_id(), charge),
1574 ))
1575 } else {
1576 Ok(HostChargeSubmissionDecision::Rejected {
1577 reason: syrup_rail::HostChargeTargetRejection::ChargeChanged,
1578 })
1579 }
1580 }
1581
1582 async fn apply_transition(
1583 &self,
1584 connection: &mut PgConnection,
1585 transition: HostChargeTargetTransition,
1586 ) -> Result<HostChargeTargetTransitionOutcome, HostChargeTargetError> {
1587 let current: Option<String> = sqlx::query_scalar(
1588 r#"
1589 SELECT status FROM host_charge_targets
1590 WHERE id = $1 AND billing_scope_id = $2 AND subscriber_id = $3
1591 FOR UPDATE
1592 "#,
1593 )
1594 .bind(transition.target_id().as_uuid())
1595 .bind(transition.billing_scope_id().as_uuid())
1596 .bind(transition.subscriber_id().as_uuid())
1597 .fetch_optional(&mut *connection)
1598 .await
1599 .map_err(HostChargeTargetError::new)?;
1600 let Some(current) = current else {
1601 return Ok(HostChargeTargetTransitionOutcome::StaleTarget);
1602 };
1603 match transition.kind() {
1604 HostChargeTargetTransitionKind::Paid if current == "pending" => {
1605 sqlx::query(
1606 "UPDATE host_charge_targets SET status = 'paid', paid_at = $2 WHERE id = $1",
1607 )
1608 .bind(transition.target_id().as_uuid())
1609 .bind(transition.effective_at())
1610 .execute(connection)
1611 .await
1612 .map_err(HostChargeTargetError::new)?;
1613 Ok(HostChargeTargetTransitionOutcome::Applied)
1614 }
1615 HostChargeTargetTransitionKind::Paid if current == "paid" => {
1616 Ok(HostChargeTargetTransitionOutcome::ExactReplay)
1617 }
1618 HostChargeTargetTransitionKind::Paid => {
1619 Ok(HostChargeTargetTransitionOutcome::StaleTarget)
1620 }
1621 HostChargeTargetTransitionKind::ReleasedBeforeSubmission
1622 | HostChargeTargetTransitionKind::PaymentFailed
1623 if current == "pending" =>
1624 {
1625 Ok(HostChargeTargetTransitionOutcome::Applied)
1626 }
1627 _ => Ok(HostChargeTargetTransitionOutcome::Unchanged {
1628 reason: HostChargeTargetNoChange::InapplicableState,
1629 }),
1630 }
1631 }
1632 }
1633
1634 struct RefusingTransitionTargets;
1635
1636 #[async_trait]
1637 impl HostChargeTargetStore for RefusingTransitionTargets {
1638 async fn preflight_target(
1639 &self,
1640 connection: &mut PgConnection,
1641 reservation: &HostChargeTargetReservation,
1642 ) -> Result<HostChargeReservationDecision, HostChargeTargetError> {
1643 TestTargets.preflight_target(connection, reservation).await
1644 }
1645
1646 async fn reserve_target(
1647 &self,
1648 connection: &mut PgConnection,
1649 reservation: &HostChargeTargetReservation,
1650 ) -> Result<HostChargeReservationDecision, HostChargeTargetError> {
1651 TestTargets.reserve_target(connection, reservation).await
1652 }
1653
1654 async fn ensure_submission_admitted(
1655 &self,
1656 connection: &mut PgConnection,
1657 admission: &HostChargeSubmissionAdmission,
1658 ) -> Result<HostChargeSubmissionDecision, HostChargeTargetError> {
1659 TestTargets
1660 .ensure_submission_admitted(connection, admission)
1661 .await
1662 }
1663
1664 async fn apply_transition(
1665 &self,
1666 _connection: &mut PgConnection,
1667 _transition: HostChargeTargetTransition,
1668 ) -> Result<HostChargeTargetTransitionOutcome, HostChargeTargetError> {
1669 Ok(HostChargeTargetTransitionOutcome::Unchanged {
1670 reason: HostChargeTargetNoChange::ReleaseUnsafe,
1671 })
1672 }
1673 }
1674
1675 fn resolved_gateway(
1676 account: crate::test_support::GatewayAccountFixture,
1677 gateway: Arc<dyn PaymentGateway>,
1678 ) -> ResolvedGateway {
1679 ResolvedGateway::new(
1680 syrup_rail::BillingScopeId::new(account.billing_scope_id),
1681 GatewayAccountId::new(account.gateway_account_id),
1682 GatewayConfigurationId::new(account.gateway_configuration_id),
1683 GatewayProviderKey::new("nmi").unwrap(),
1684 GatewayLifecycleQueryPolicy::new(
1685 GatewayLifecycleCursorKey::new("host_test").unwrap(),
1686 ChronoDuration::minutes(1),
1687 10,
1688 2,
1689 2,
1690 20,
1691 )
1692 .unwrap(),
1693 Arc::new(TestReferenceFactory),
1694 gateway,
1695 )
1696 }
1697
1698 fn approved_outcome(transaction_id: &str) -> GatewayPaymentOutcome {
1699 GatewayPaymentOutcome::new(
1700 GatewayPaymentStatus::Approved,
1701 ProcessorEvidence::new(
1702 Some(GatewayTransactionId::new(transaction_id).unwrap()),
1703 None,
1704 Some(GatewayDiagnostic::new("1")),
1705 None,
1706 Some(GatewayDiagnostic::new("approved")),
1707 Some(GatewayDiagnostic::new("complete")),
1708 GatewayPaymentDescriptor::default(),
1709 ),
1710 )
1711 }
1712
1713 #[tokio::test]
1714 async fn foreground_host_charge_is_one_shot_atomic_and_replay_first()
1715 -> Result<(), Box<dyn Error>> {
1716 let database = TestDatabase::start("rail_host_svc").await?;
1717 let result = async {
1718 sqlx::query(
1719 r#"
1720 CREATE TABLE host_charge_targets (
1721 id uuid PRIMARY KEY,
1722 billing_scope_id uuid NOT NULL,
1723 subscriber_id uuid NOT NULL,
1724 status text NOT NULL,
1725 amount_cents integer NOT NULL,
1726 currency text NOT NULL,
1727 paid_at timestamptz
1728 )
1729 "#,
1730 )
1731 .execute(&database.pool)
1732 .await?;
1733 let account = create_gateway_account(&database.pool, "nmi").await?;
1734 let subscriber_id = Uuid::now_v7();
1735 let target_id = Uuid::now_v7();
1736 sqlx::query(
1737 "INSERT INTO host_charge_targets VALUES ($1, $2, $3, 'pending', 1250, 'USD', NULL)",
1738 )
1739 .bind(target_id)
1740 .bind(account.billing_scope_id)
1741 .bind(subscriber_id)
1742 .execute(&database.pool)
1743 .await?;
1744
1745 let gateway = Arc::new(ScriptedGateway {
1746 account_mode: GatewayAccountMode::Test,
1747 sale_calls: AtomicUsize::new(0),
1748 outcome: Mutex::new(Some(
1749 approved_outcome("host_txn_approved").with_diagnostics(vec![
1750 GatewayPaymentDiagnostic::ProcessorReportedDuplicate,
1751 ]),
1752 )),
1753 });
1754 let resolver = Arc::new(StaticResolver {
1755 gateway: resolved_gateway(account, gateway.clone()),
1756 calls: AtomicUsize::new(0),
1757 });
1758 let admission = Arc::new(PermitAdmission {
1759 calls: AtomicUsize::new(0),
1760 });
1761 let events = Arc::new(Mutex::new(Vec::new()));
1762 let coordinator = Arc::new(TestCoordinator {
1763 pool: database.pool.clone(),
1764 events: Arc::clone(&events),
1765 });
1766 let targets = Arc::new(TestTargets);
1767 let service = SubscriptionBillingService::new(
1768 database.pool.clone(),
1769 Arc::new(UnusedOffers),
1770 resolver.clone(),
1771 admission.clone(),
1772 coordinator.clone(),
1773 )
1774 .with_required_gateway_account_mode(GatewayAccountMode::Test)
1775 .with_host_charge_targets(targets);
1776 let command = ChargeHostTarget::new(
1777 syrup_rail::BillingScopeId::new(account.billing_scope_id),
1778 syrup_rail::SubscriberId::new(subscriber_id),
1779 HostChargeTargetId::new(target_id),
1780 GatewayConfigurationId::new(account.gateway_configuration_id),
1781 PaymentToken::new("tok_host_once")?,
1782 IdempotencyKey::new("host-idempotency")?,
1783 Some(BillingContact::new(
1784 None,
1785 None,
1786 Some("host@example.test".into()),
1787 )?),
1788 );
1789
1790 let first = service.charge_host_target(command.clone()).await?;
1791 assert_eq!(first.status(), PaymentAttemptStatus::Approved);
1792 assert_eq!(
1793 first.gateway_diagnostics(),
1794 &[GatewayPaymentDiagnostic::ProcessorReportedDuplicate]
1795 );
1796 assert_eq!(gateway.sale_calls.load(Ordering::SeqCst), 1);
1797 assert_eq!(resolver.calls.load(Ordering::SeqCst), 1);
1798 assert_eq!(admission.calls.load(Ordering::SeqCst), 1);
1799 sqlx::query("UPDATE host_charge_targets SET status = 'reversed' WHERE id = $1")
1800 .bind(target_id)
1801 .execute(&database.pool)
1802 .await?;
1803 let approved_replay = approved_outcome("host_txn_approved")
1804 .with_diagnostics(vec![GatewayPaymentDiagnostic::ProcessorReportedDuplicate]);
1805 let reconciled_replay = apply_reconciled_host_charge_gateway_outcome(
1806 &database.pool,
1807 coordinator.as_ref(),
1808 &RefusingTransitionTargets,
1809 syrup_rail::BillingScopeId::new(account.billing_scope_id),
1810 first.attempt().identity().attempt_id(),
1811 &approved_replay,
1812 )
1813 .await?;
1814 assert_eq!(reconciled_replay.attempt(), first.attempt());
1815 assert_eq!(
1816 reconciled_replay.gateway_diagnostics(),
1817 &[GatewayPaymentDiagnostic::ProcessorReportedDuplicate]
1818 );
1819 let live_service = service
1820 .clone()
1821 .with_required_gateway_account_mode(GatewayAccountMode::Live);
1822 let replay = live_service.charge_host_target(command).await?;
1823 assert_eq!(replay.attempt(), first.attempt());
1824 assert!(replay.gateway_diagnostics().is_empty());
1825 assert_eq!(gateway.sale_calls.load(Ordering::SeqCst), 1);
1826 assert_eq!(resolver.calls.load(Ordering::SeqCst), 1);
1827 assert_eq!(admission.calls.load(Ordering::SeqCst), 1);
1828
1829 let target_status: String =
1830 sqlx::query_scalar("SELECT status FROM host_charge_targets WHERE id = $1")
1831 .bind(target_id)
1832 .fetch_one(&database.pool)
1833 .await?;
1834 assert_eq!(target_status, "reversed");
1835 let charge_state: String = sqlx::query_scalar(
1836 "SELECT progression_state FROM billing_processor_charges WHERE attempt_id = $1",
1837 )
1838 .bind(first.attempt().identity().attempt_id().as_uuid())
1839 .fetch_one(&database.pool)
1840 .await?;
1841 assert_eq!(charge_state, "applied");
1842 let event_keys = events
1843 .lock()
1844 .await
1845 .iter()
1846 .map(BillingEvent::semantic_key)
1847 .collect::<Vec<_>>();
1848 assert_eq!(
1849 event_keys,
1850 vec![BillingEventKey::HostChargePaid(HostChargeTargetId::new(
1851 target_id
1852 ))]
1853 );
1854 Ok::<_, Box<dyn Error>>(())
1855 }
1856 .await;
1857 let cleanup = database.cleanup().await;
1858 result?;
1859 cleanup?;
1860 Ok(())
1861 }
1862
1863 #[tokio::test]
1864 async fn submission_admission_validates_identity_and_skips_nonprepared_replay()
1865 -> Result<(), Box<dyn Error>> {
1866 let database = TestDatabase::start("rail_host_ident").await?;
1867 let result = async {
1868 sqlx::query(
1869 r#"
1870 CREATE TABLE host_charge_targets (
1871 id uuid PRIMARY KEY,
1872 billing_scope_id uuid NOT NULL,
1873 subscriber_id uuid NOT NULL,
1874 status text NOT NULL,
1875 amount_cents integer NOT NULL,
1876 currency text NOT NULL,
1877 paid_at timestamptz
1878 )
1879 "#,
1880 )
1881 .execute(&database.pool)
1882 .await?;
1883 let account = create_gateway_account(&database.pool, "nmi").await?;
1884 let subscriber_id = Uuid::now_v7();
1885 let target_id = Uuid::now_v7();
1886 sqlx::query(
1887 "INSERT INTO host_charge_targets VALUES ($1, $2, $3, 'pending', 1250, 'USD', NULL)",
1888 )
1889 .bind(target_id)
1890 .bind(account.billing_scope_id)
1891 .bind(subscriber_id)
1892 .execute(&database.pool)
1893 .await?;
1894 let gateway = resolved_gateway(
1895 account,
1896 Arc::new(ScriptedGateway {
1897 account_mode: GatewayAccountMode::Live,
1898 sale_calls: AtomicUsize::new(0),
1899 outcome: Mutex::new(None),
1900 }),
1901 );
1902 let snapshot = syrup_rail::HostChargeTargetSnapshot::new(
1903 HostChargeTargetId::new(target_id),
1904 ChargeAmount::new(1250, CurrencyCode::new("USD")?)?,
1905 );
1906 let command = ChargeHostTarget::new(
1907 syrup_rail::BillingScopeId::new(account.billing_scope_id),
1908 syrup_rail::SubscriberId::new(subscriber_id),
1909 HostChargeTargetId::new(target_id),
1910 GatewayConfigurationId::new(account.gateway_configuration_id),
1911 PaymentToken::new("tok_host_admission_identity")?,
1912 IdempotencyKey::new("host-admission-identity")?,
1913 None,
1914 );
1915 let attempt_id = PaymentAttemptId::new(Uuid::now_v7());
1916 let reservation = HostChargeReservation::from_command(
1917 &command,
1918 snapshot,
1919 &gateway,
1920 attempt_id,
1921 GatewayAccountMode::Live,
1922 )?;
1923 let mut transaction = database.pool.begin().await?;
1924 assert!(matches!(
1925 reserve_host_charge_in_transaction(&mut transaction, &TestTargets, &reservation)
1926 .await?,
1927 HostChargeReservationOutcome::Reserved(_)
1928 ));
1929 transaction.commit().await?;
1930
1931 let wrong_mode = HostChargeReservation::from_command(
1932 &command,
1933 snapshot,
1934 &gateway,
1935 attempt_id,
1936 GatewayAccountMode::Test,
1937 )?;
1938 assert!(matches!(
1939 admit_host_charge_submission(&database.pool, &TestTargets, &wrong_mode).await,
1940 Err(HostChargeApplicationError::Store(
1941 HostChargeStoreError::InvalidState
1942 ))
1943 ));
1944
1945 let mut transaction = database.pool.begin().await?;
1946 let attempt = crate::find_payment_attempt_by_id_in_transaction(
1947 &mut transaction,
1948 command.billing_scope_id(),
1949 attempt_id,
1950 )
1951 .await?
1952 .expect("reserved attempt");
1953 transaction.commit().await?;
1954 assert_eq!(attempt.status(), PaymentAttemptStatus::Pending);
1955 assert!(attempt.state().timestamps().submitted_at().is_none());
1956
1957 sqlx::query(
1958 "UPDATE billing_payment_attempts SET status = 'review_required' WHERE id = $1",
1959 )
1960 .bind(attempt_id.as_uuid())
1961 .execute(&database.pool)
1962 .await?;
1963 let HostChargeAdmissionOutcome::AlreadyAdmitted(review) =
1964 admit_host_charge_submission(&database.pool, &AdmissionMustNotRun, &wrong_mode)
1965 .await?
1966 else {
1967 panic!("wrong-mode review replay must return the canonical attempt");
1968 };
1969 assert_eq!(review.status(), PaymentAttemptStatus::ReviewRequired);
1970 Ok::<_, Box<dyn Error>>(())
1971 }
1972 .await;
1973 let cleanup = database.cleanup().await;
1974 result?;
1975 cleanup?;
1976 Ok(())
1977 }
1978
1979 #[tokio::test]
1980 async fn reservation_race_replays_equivalent_contact_and_rejects_changed_contact()
1981 -> Result<(), Box<dyn Error>> {
1982 let database = TestDatabase::start("rail_host_race").await?;
1983 let result = async {
1984 sqlx::query(
1985 r#"
1986 CREATE TABLE host_charge_targets (
1987 id uuid PRIMARY KEY,
1988 billing_scope_id uuid NOT NULL,
1989 subscriber_id uuid NOT NULL,
1990 status text NOT NULL,
1991 amount_cents integer NOT NULL,
1992 currency text NOT NULL,
1993 paid_at timestamptz
1994 )
1995 "#,
1996 )
1997 .execute(&database.pool)
1998 .await?;
1999 let account = create_gateway_account(&database.pool, "nmi").await?;
2000 let subscriber_id = Uuid::now_v7();
2001 let target_id = Uuid::now_v7();
2002 sqlx::query(
2003 "INSERT INTO host_charge_targets VALUES ($1, $2, $3, 'pending', 1250, 'USD', NULL)",
2004 )
2005 .bind(target_id)
2006 .bind(account.billing_scope_id)
2007 .bind(subscriber_id)
2008 .execute(&database.pool)
2009 .await?;
2010 let gateway = resolved_gateway(
2011 account,
2012 Arc::new(ScriptedGateway {
2013 account_mode: GatewayAccountMode::Live,
2014 sale_calls: AtomicUsize::new(0),
2015 outcome: Mutex::new(None),
2016 }),
2017 );
2018 let snapshot = syrup_rail::HostChargeTargetSnapshot::new(
2019 HostChargeTargetId::new(target_id),
2020 ChargeAmount::new(1250, CurrencyCode::new("USD")?)?,
2021 );
2022 let command = ChargeHostTarget::new(
2023 syrup_rail::BillingScopeId::new(account.billing_scope_id),
2024 syrup_rail::SubscriberId::new(subscriber_id),
2025 HostChargeTargetId::new(target_id),
2026 GatewayConfigurationId::new(account.gateway_configuration_id),
2027 PaymentToken::new("tok_host_winner")?,
2028 IdempotencyKey::new("host-reservation-race")?,
2029 Some(BillingContact::new(
2030 Some("Mary Ann".into()),
2031 Some("Smith".into()),
2032 Some("winner@example.test".into()),
2033 )?),
2034 );
2035 let winner_id = PaymentAttemptId::new(Uuid::now_v7());
2036 let winner = HostChargeReservation::from_command(
2037 &command,
2038 snapshot,
2039 &gateway,
2040 winner_id,
2041 GatewayAccountMode::Live,
2042 )?;
2043 let mut transaction = database.pool.begin().await?;
2044 let outcome =
2045 reserve_host_charge_in_transaction(&mut transaction, &TestTargets, &winner).await?;
2046 assert!(matches!(outcome, HostChargeReservationOutcome::Reserved(_)));
2047 transaction.commit().await?;
2048
2049 let retry = ChargeHostTarget::new(
2050 command.billing_scope_id(),
2051 command.subscriber_id(),
2052 command.target_id(),
2053 command.gateway_configuration_id(),
2054 PaymentToken::new("tok_host_retry")?,
2055 command.idempotency_key().clone(),
2056 command.billing_contact().cloned(),
2057 );
2058 let contender = HostChargeReservation::from_command(
2059 &retry,
2060 snapshot,
2061 &gateway,
2062 PaymentAttemptId::new(Uuid::now_v7()),
2063 GatewayAccountMode::Live,
2064 )?;
2065 let mut transaction = database.pool.begin().await?;
2066 let outcome =
2067 reserve_host_charge_in_transaction(&mut transaction, &TestTargets, &contender)
2068 .await?;
2069 transaction.commit().await?;
2070 let HostChargeReservationOutcome::Replay(attempt) = outcome else {
2071 panic!("matching contender should replay the durable winner");
2072 };
2073 assert_eq!(attempt.identity().attempt_id(), winner_id);
2074 assert_eq!(
2075 attempt.request().billing_contact().email(),
2076 Some("winner@example.test")
2077 );
2078
2079 let changed_mode_contender = HostChargeReservation::from_command(
2080 &retry,
2081 snapshot,
2082 &gateway,
2083 PaymentAttemptId::new(Uuid::now_v7()),
2084 GatewayAccountMode::Test,
2085 )?;
2086 let mut transaction = database.pool.begin().await?;
2087 let outcome = reserve_host_charge_in_transaction(
2088 &mut transaction,
2089 &TestTargets,
2090 &changed_mode_contender,
2091 )
2092 .await?;
2093 transaction.rollback().await?;
2094 assert_eq!(
2095 outcome,
2096 HostChargeReservationOutcome::GatewayAccountModeChanged
2097 );
2098
2099 sqlx::query(
2100 "UPDATE billing_payment_attempts SET status = 'review_required' WHERE id = $1",
2101 )
2102 .bind(winner_id.as_uuid())
2103 .execute(&database.pool)
2104 .await?;
2105 let mut transaction = database.pool.begin().await?;
2106 let outcome = reserve_host_charge_in_transaction(
2107 &mut transaction,
2108 &TestTargets,
2109 &changed_mode_contender,
2110 )
2111 .await?;
2112 transaction.commit().await?;
2113 let HostChargeReservationOutcome::Replay(review) = outcome else {
2114 panic!("an unsubmitted review attempt cannot resume provider submission");
2115 };
2116 assert_eq!(review.identity().attempt_id(), winner_id);
2117 assert_eq!(review.status(), PaymentAttemptStatus::ReviewRequired);
2118
2119 let changed_contact_retry = ChargeHostTarget::new(
2120 command.billing_scope_id(),
2121 command.subscriber_id(),
2122 command.target_id(),
2123 command.gateway_configuration_id(),
2124 PaymentToken::new("tok_host_changed_contact")?,
2125 command.idempotency_key().clone(),
2126 Some(BillingContact::new(
2127 Some("Mary".into()),
2128 Some("Ann Smith".into()),
2129 Some("winner@example.test".into()),
2130 )?),
2131 );
2132 let changed_contact_contender = HostChargeReservation::from_command(
2133 &changed_contact_retry,
2134 snapshot,
2135 &gateway,
2136 PaymentAttemptId::new(Uuid::now_v7()),
2137 GatewayAccountMode::Live,
2138 )?;
2139 let mut transaction = database.pool.begin().await?;
2140 let outcome = reserve_host_charge_in_transaction(
2141 &mut transaction,
2142 &TestTargets,
2143 &changed_contact_contender,
2144 )
2145 .await?;
2146 transaction.rollback().await?;
2147 assert_eq!(outcome, HostChargeReservationOutcome::IdempotencyConflict);
2148 Ok::<_, Box<dyn Error>>(())
2149 }
2150 .await;
2151 let cleanup = database.cleanup().await;
2152 result?;
2153 cleanup?;
2154 Ok(())
2155 }
2156
2157 #[tokio::test]
2158 async fn unreserved_rate_limit_avoids_attempt_and_extends_provider_cooldown()
2159 -> Result<(), Box<dyn Error>> {
2160 let database = TestDatabase::start("rail_host_rate").await?;
2161 let result = async {
2162 sqlx::query(
2163 r#"
2164 CREATE TABLE host_charge_targets (
2165 id uuid PRIMARY KEY,
2166 billing_scope_id uuid NOT NULL,
2167 subscriber_id uuid NOT NULL,
2168 status text NOT NULL,
2169 amount_cents integer NOT NULL,
2170 currency text NOT NULL,
2171 paid_at timestamptz
2172 )
2173 "#,
2174 )
2175 .execute(&database.pool)
2176 .await?;
2177 let account = create_gateway_account(&database.pool, "nmi").await?;
2178 let subscriber_id = Uuid::now_v7();
2179 let target_id = Uuid::now_v7();
2180 sqlx::query(
2181 "INSERT INTO host_charge_targets VALUES ($1, $2, $3, 'pending', 1250, 'USD', NULL)",
2182 )
2183 .bind(target_id)
2184 .bind(account.billing_scope_id)
2185 .bind(subscriber_id)
2186 .execute(&database.pool)
2187 .await?;
2188
2189 let gateway = Arc::new(RateLimitedAfterReservationGateway {
2190 readiness_calls: AtomicUsize::new(0),
2191 sale_calls: AtomicUsize::new(0),
2192 });
2193 let resolver = Arc::new(StaticResolver {
2194 gateway: resolved_gateway(account, gateway.clone()),
2195 calls: AtomicUsize::new(0),
2196 });
2197 let admission = Arc::new(PermitAdmission {
2198 calls: AtomicUsize::new(0),
2199 });
2200 let service = SubscriptionBillingService::new(
2201 database.pool.clone(),
2202 Arc::new(UnusedOffers),
2203 resolver,
2204 admission.clone(),
2205 Arc::new(TestCoordinator {
2206 pool: database.pool.clone(),
2207 events: Arc::new(Mutex::new(Vec::new())),
2208 }),
2209 )
2210 .with_host_charge_targets(Arc::new(TestTargets));
2211 let command = ChargeHostTarget::new(
2212 syrup_rail::BillingScopeId::new(account.billing_scope_id),
2213 syrup_rail::SubscriberId::new(subscriber_id),
2214 HostChargeTargetId::new(target_id),
2215 GatewayConfigurationId::new(account.gateway_configuration_id),
2216 PaymentToken::new("tok_host_throttled")?,
2217 IdempotencyKey::new("host-throttled")?,
2218 None,
2219 );
2220
2221 let error = service
2222 .charge_host_target(command)
2223 .await
2224 .expect_err("provider throttle must be reported");
2225 assert!(matches!(
2226 error,
2227 crate::SubscriptionBillingServiceError::GatewayMutationCooldown {
2228 scope: crate::GatewayMutationCooldownScope::Provider
2229 }
2230 ));
2231 assert_eq!(gateway.readiness_calls.load(Ordering::SeqCst), 1);
2232 assert_eq!(gateway.sale_calls.load(Ordering::SeqCst), 0);
2233
2234 let attempt_count: i64 = sqlx::query_scalar(
2235 "SELECT COUNT(*) FROM billing_payment_attempts \
2236 WHERE billing_scope_id = $1 AND subscriber_id = $2",
2237 )
2238 .bind(account.billing_scope_id)
2239 .bind(subscriber_id)
2240 .fetch_one(&database.pool)
2241 .await?;
2242 assert_eq!(attempt_count, 0);
2243 assert_eq!(admission.calls.load(Ordering::SeqCst), 0);
2244 let cooldown_is_active: bool = sqlx::query_scalar(
2245 "SELECT rate_limited_until > clock_timestamp() FROM billing_gateway_provider_rate_limits WHERE provider_key = 'nmi'",
2246 )
2247 .fetch_one(&database.pool)
2248 .await?;
2249 assert!(cooldown_is_active);
2250 let target_status: String =
2251 sqlx::query_scalar("SELECT status FROM host_charge_targets WHERE id = $1")
2252 .bind(target_id)
2253 .fetch_one(&database.pool)
2254 .await?;
2255 assert_eq!(target_status, "pending");
2256 Ok::<_, Box<dyn Error>>(())
2257 }
2258 .await;
2259 let cleanup = database.cleanup().await;
2260 result?;
2261 cleanup?;
2262 Ok(())
2263 }
2264
2265 #[tokio::test]
2266 async fn terminal_approval_race_does_not_mark_host_target_paid() -> Result<(), Box<dyn Error>> {
2267 let database = TestDatabase::start("rail_host_race").await?;
2268 let result = async {
2269 sqlx::query(
2270 r#"
2271 CREATE TABLE host_charge_targets (
2272 id uuid PRIMARY KEY,
2273 billing_scope_id uuid NOT NULL,
2274 subscriber_id uuid NOT NULL,
2275 status text NOT NULL,
2276 amount_cents integer NOT NULL,
2277 currency text NOT NULL,
2278 paid_at timestamptz
2279 )
2280 "#,
2281 )
2282 .execute(&database.pool)
2283 .await?;
2284 let account = create_gateway_account(&database.pool, "nmi").await?;
2285 let subscriber_id = Uuid::now_v7();
2286 let target_id = Uuid::now_v7();
2287 sqlx::query(
2288 "INSERT INTO host_charge_targets VALUES ($1, $2, $3, 'pending', 1250, 'USD', NULL)",
2289 )
2290 .bind(target_id)
2291 .bind(account.billing_scope_id)
2292 .bind(subscriber_id)
2293 .execute(&database.pool)
2294 .await?;
2295
2296 let gateway = Arc::new(TerminalRaceGateway {
2297 pool: database.pool.clone(),
2298 sale_calls: AtomicUsize::new(0),
2299 });
2300 let resolver = Arc::new(StaticResolver {
2301 gateway: resolved_gateway(account, gateway.clone()),
2302 calls: AtomicUsize::new(0),
2303 });
2304 let events = Arc::new(Mutex::new(Vec::new()));
2305 let service = SubscriptionBillingService::new(
2306 database.pool.clone(),
2307 Arc::new(UnusedOffers),
2308 resolver,
2309 Arc::new(PermitAdmission {
2310 calls: AtomicUsize::new(0),
2311 }),
2312 Arc::new(TestCoordinator {
2313 pool: database.pool.clone(),
2314 events: Arc::clone(&events),
2315 }),
2316 )
2317 .with_host_charge_targets(Arc::new(TestTargets));
2318 let payment = service
2319 .charge_host_target(ChargeHostTarget::new(
2320 syrup_rail::BillingScopeId::new(account.billing_scope_id),
2321 syrup_rail::SubscriberId::new(subscriber_id),
2322 HostChargeTargetId::new(target_id),
2323 GatewayConfigurationId::new(account.gateway_configuration_id),
2324 PaymentToken::new("tok_host_race")?,
2325 IdempotencyKey::new("host-race")?,
2326 None,
2327 ))
2328 .await?;
2329
2330 assert_eq!(payment.status(), PaymentAttemptStatus::Unknown);
2331 assert_eq!(payment.attempt().status(), PaymentAttemptStatus::Failed);
2332 assert_eq!(gateway.sale_calls.load(Ordering::SeqCst), 1);
2333 assert!(events.lock().await.is_empty());
2334 let target_status: String =
2335 sqlx::query_scalar("SELECT status FROM host_charge_targets WHERE id = $1")
2336 .bind(target_id)
2337 .fetch_one(&database.pool)
2338 .await?;
2339 assert_eq!(target_status, "pending");
2340 let charge_state: String = sqlx::query_scalar(
2341 "SELECT progression_state FROM billing_processor_charges WHERE attempt_id = $1",
2342 )
2343 .bind(payment.attempt().identity().attempt_id().as_uuid())
2344 .fetch_one(&database.pool)
2345 .await?;
2346 assert_eq!(charge_state, "pending");
2347 Ok::<_, Box<dyn Error>>(())
2348 }
2349 .await;
2350 let cleanup = database.cleanup().await;
2351 result?;
2352 cleanup?;
2353 Ok(())
2354 }
2355
2356 #[tokio::test]
2357 async fn matching_retry_resumes_prepared_attempt_without_readiness_loser_overwrite()
2358 -> Result<(), Box<dyn Error>> {
2359 let database = TestDatabase::start("rail_host_resume").await?;
2360 let result = async {
2361 sqlx::query(
2362 r#"
2363 CREATE TABLE host_charge_targets (
2364 id uuid PRIMARY KEY,
2365 billing_scope_id uuid NOT NULL,
2366 subscriber_id uuid NOT NULL,
2367 status text NOT NULL,
2368 amount_cents integer NOT NULL,
2369 currency text NOT NULL,
2370 paid_at timestamptz
2371 )
2372 "#,
2373 )
2374 .execute(&database.pool)
2375 .await?;
2376 let account = create_gateway_account(&database.pool, "nmi").await?;
2377 let subscriber_id = Uuid::now_v7();
2378 let target_id = Uuid::now_v7();
2379 sqlx::query(
2380 "INSERT INTO host_charge_targets VALUES ($1, $2, $3, 'pending', 1250, 'USD', NULL)",
2381 )
2382 .bind(target_id)
2383 .bind(account.billing_scope_id)
2384 .bind(subscriber_id)
2385 .execute(&database.pool)
2386 .await?;
2387
2388 let (readiness_started_tx, readiness_started_rx) = oneshot::channel();
2389 let (sale_started_tx, sale_started_rx) = oneshot::channel();
2390 let gateway = Arc::new(RacingPreparedRetryGateway {
2391 readiness_calls: AtomicUsize::new(0),
2392 sale_calls: AtomicUsize::new(0),
2393 blocked_readiness_started: Mutex::new(Some(readiness_started_tx)),
2394 sale_started: Mutex::new(Some(sale_started_tx)),
2395 release_readiness: Notify::new(),
2396 release_sale: Notify::new(),
2397 });
2398 let command = ChargeHostTarget::new(
2399 syrup_rail::BillingScopeId::new(account.billing_scope_id),
2400 syrup_rail::SubscriberId::new(subscriber_id),
2401 HostChargeTargetId::new(target_id),
2402 GatewayConfigurationId::new(account.gateway_configuration_id),
2403 PaymentToken::new("tok_host_resume")?,
2404 IdempotencyKey::new("host-resume")?,
2405 None,
2406 );
2407 let resolved = resolved_gateway(account, gateway.clone());
2408 let reservation = HostChargeReservation::from_command(
2409 &command,
2410 syrup_rail::HostChargeTargetSnapshot::new(
2411 HostChargeTargetId::new(target_id),
2412 ChargeAmount::new(1250, CurrencyCode::new("USD")?)?,
2413 ),
2414 &resolved,
2415 PaymentAttemptId::new(Uuid::now_v7()),
2416 GatewayAccountMode::Live,
2417 )?;
2418 let mut transaction = database.pool.begin().await?;
2419 assert!(matches!(
2420 reserve_host_charge_in_transaction(&mut transaction, &TestTargets, &reservation,)
2421 .await?,
2422 HostChargeReservationOutcome::Reserved(_)
2423 ));
2424 transaction.commit().await?;
2425 let service = SubscriptionBillingService::new(
2426 database.pool.clone(),
2427 Arc::new(UnusedOffers),
2428 Arc::new(StaticResolver {
2429 gateway: resolved,
2430 calls: AtomicUsize::new(0),
2431 }),
2432 Arc::new(PermitAdmission {
2433 calls: AtomicUsize::new(0),
2434 }),
2435 Arc::new(TestCoordinator {
2436 pool: database.pool.clone(),
2437 events: Arc::new(Mutex::new(Vec::new())),
2438 }),
2439 )
2440 .with_host_charge_targets(Arc::new(TestTargets));
2441
2442 let first_service = service.clone();
2443 let first_command = command.clone();
2444 let first =
2445 tokio::spawn(async move { first_service.charge_host_target(first_command).await });
2446 readiness_started_rx.await?;
2447 let retry = ChargeHostTarget::new(
2448 command.billing_scope_id(),
2449 command.subscriber_id(),
2450 command.target_id(),
2451 command.gateway_configuration_id(),
2452 PaymentToken::new("tok_host_resume_retry")?,
2453 command.idempotency_key().clone(),
2454 command.billing_contact().cloned(),
2455 );
2456 let second_service = service.clone();
2457 let second =
2458 tokio::spawn(async move { second_service.charge_host_target(retry).await });
2459 sale_started_rx.await?;
2460 gateway.release_readiness.notify_one();
2461 let first = first.await??;
2462 assert_eq!(first.status(), PaymentAttemptStatus::Pending);
2463 assert!(
2464 first
2465 .attempt()
2466 .state()
2467 .timestamps()
2468 .submitted_at()
2469 .is_some()
2470 );
2471 gateway.release_sale.notify_one();
2472 let second = second.await??;
2473 assert_eq!(second.status(), PaymentAttemptStatus::Approved);
2474 assert_eq!(first.attempt().identity(), second.attempt().identity());
2475 assert_eq!(gateway.sale_calls.load(Ordering::SeqCst), 1);
2476 Ok::<_, Box<dyn Error>>(())
2477 }
2478 .await;
2479 let cleanup = database.cleanup().await;
2480 result?;
2481 cleanup?;
2482 Ok(())
2483 }
2484}