Skip to main content

syrup_rail_postgres/
host_charge_application.rs

1use std::{fmt, time::Duration};
2
3use chrono::{DateTime, Utc};
4use sqlx::{PgConnection, PgPool};
5use syrup_rail::{
6    ApprovedProcessorEvidence, BillingEvent, BillingEventSubject, ChargeHostTarget,
7    GatewayMutationError, GatewayNotSubmittedError, GatewayPaymentOutcome, GatewayPaymentStatus,
8    GatewayProviderKey, GatewaySaleIntent, GatewaySaleRequest, HostChargePaymentResult,
9    HostChargePaymentResultBuildError, HostChargeReservation, HostChargeTargetTransition,
10    HostChargeTargetTransitionKind, HostChargeTargetTransitionOutcome, PaymentAttempt,
11    PaymentAttemptStatus, PaymentResolutionCode, ProcessorChargeProgression, ProcessorChargeRole,
12    ProcessorEvidence,
13};
14use thiserror::Error;
15
16use crate::host_charges::host_charge_attempt_matches_reservation;
17use crate::{
18    BillingTransactionCoordinator, BillingTransactionError, BillingTransactionSubjectState,
19    HostChargeStoreError, HostChargeSubmissionOutcome, HostChargeTargetError,
20    HostChargeTargetStore, ModeVerifiedGateway, ProcessorChargeStoreError,
21    admit_host_charge_submission_in_transaction,
22    attempts::{
23        AttemptApproval, AttemptResolutionStatus, AttemptTransition, PaymentAttemptStoreError,
24        find_payment_attempt_by_id_on_connection, lock_payment_attempt_by_id_on_connection,
25        persist_attempt_transition,
26    },
27    enrollment_application::{
28        GatewayNotSubmittedPolicy, OutcomeResolutionBoundary, PreparedAttemptReplay,
29        RateLimitCooldown, RateLimitCooldownCommitError, RateLimitCooldownOperation,
30        SubscriptionEnrollmentApplicationError, commit_rate_limit_cooldown_for_operation,
31        map_attempt_transition_error, mutation_error_evidence, park_locked_attempt,
32        restore_prepared_attempt_submission, set_application_timeouts,
33        should_surface_not_submitted_application,
34    },
35    processor_charges::{ObservedCharge, observe_processor_charge, transition_charge},
36};
37
38const BILLING_LOCK_TIMEOUT: Duration = Duration::from_millis(250);
39const APPROVED_APPLICATION_ATTEMPTS: usize = 3;
40const APPROVED_EVIDENCE_RETRY_DELAY: Duration = Duration::from_millis(50);
41const INVALID_HOST_CHARGE_STATE: &str = "canonical host charge application state is invalid";
42const APPROVED_STALE_TARGET_TEXT: &str =
43    "Approved host charge could not update its target because the target changed.";
44const APPROVED_STORAGE_FAILURE_TEXT: &str =
45    "Approved host charge could not be applied; manual review is required.";
46
47#[derive(Debug, Error)]
48pub enum HostChargeApplicationError {
49    #[error("host charge application storage failed")]
50    Sql(#[from] sqlx::Error),
51    #[error("host charge attempt storage failed")]
52    Attempt(#[from] PaymentAttemptStoreError),
53    #[error("host charge target operation failed")]
54    Target(#[from] HostChargeTargetError),
55    #[error("host charge storage operation failed")]
56    Store(#[from] HostChargeStoreError),
57    #[error("host charge processor evidence operation failed")]
58    ProcessorCharge(#[from] ProcessorChargeStoreError),
59    #[error("host billing transaction failed")]
60    Transaction(#[from] BillingTransactionError),
61    #[error("host billing event append failed")]
62    Event(#[from] crate::BillingEventWriteError),
63    #[error("shared payment application failed")]
64    SharedApplication(#[from] SubscriptionEnrollmentApplicationError),
65    #[error("admitted host charge does not match the submission command or gateway")]
66    SubmissionIdentityMismatch,
67    #[error("{0}")]
68    InvalidState(&'static str),
69}
70
71impl From<HostChargePaymentResultBuildError> for HostChargeApplicationError {
72    fn from(_: HostChargePaymentResultBuildError) -> Self {
73        Self::InvalidState(INVALID_HOST_CHARGE_STATE)
74    }
75}
76
77pub struct AdmittedHostCharge {
78    reservation: HostChargeReservation,
79    attempt: PaymentAttempt,
80}
81
82impl AdmittedHostCharge {
83    pub const fn attempt(&self) -> &PaymentAttempt {
84        &self.attempt
85    }
86}
87
88impl fmt::Debug for AdmittedHostCharge {
89    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
90        formatter
91            .debug_struct("AdmittedHostCharge")
92            .field("attempt", &self.attempt)
93            .field("has_submission_authority", &true)
94            .finish()
95    }
96}
97
98#[derive(Debug)]
99pub enum HostChargeAdmissionOutcome {
100    Admitted(Box<AdmittedHostCharge>),
101    AlreadyAdmitted(PaymentAttempt),
102    Rejected {
103        attempt: PaymentAttempt,
104        reason: syrup_rail::HostChargeTargetRejection,
105    },
106}
107
108#[derive(Debug)]
109pub enum HostChargeProviderResult {
110    Payment(HostChargePaymentResult),
111    /// The provider mutation was not contacted. A retry-safe readiness failure
112    /// can carry the same pending, prepared payment for same-key replay. A
113    /// concurrent terminal result is returned as `Payment` instead.
114    NotSubmitted {
115        payment: HostChargePaymentResult,
116        error: GatewayNotSubmittedError,
117    },
118}
119
120struct HostChargeResolutionApplication {
121    payment: HostChargePaymentResult,
122    applied: bool,
123}
124
125impl HostChargeResolutionApplication {
126    fn into_payment(self) -> HostChargePaymentResult {
127        self.payment
128    }
129
130    fn should_surface_not_submitted(&self, policy: GatewayNotSubmittedPolicy) -> bool {
131        should_surface_not_submitted_application(
132            self.applied,
133            self.payment.attempt(),
134            policy,
135            PreparedAttemptReplay::Supported,
136        )
137    }
138}
139
140/// Complete persistence behavior for a host charge resolved before provider
141/// submission.
142#[derive(Clone, Copy)]
143pub(crate) struct HostChargeBeforeSubmissionResolution {
144    boundary: OutcomeResolutionBoundary,
145    cooldown: Option<RateLimitCooldown>,
146}
147
148#[derive(Clone, Copy)]
149struct HostChargeNonApprovedResolution<'provider> {
150    boundary: OutcomeResolutionBoundary,
151    cooldown: Option<(&'provider GatewayProviderKey, RateLimitCooldown)>,
152}
153
154impl HostChargeNonApprovedResolution<'_> {
155    const fn submitted() -> Self {
156        Self {
157            boundary: OutcomeResolutionBoundary::Submitted,
158            cooldown: None,
159        }
160    }
161}
162
163impl HostChargeBeforeSubmissionResolution {
164    pub(crate) const fn prepared() -> Self {
165        Self {
166            boundary: OutcomeResolutionBoundary::Prepared,
167            cooldown: None,
168        }
169    }
170
171    pub(crate) const fn admitted_not_submitted() -> Self {
172        Self {
173            boundary: OutcomeResolutionBoundary::AdmittedNotSubmitted,
174            cooldown: None,
175        }
176    }
177
178    pub(crate) const fn with_not_submitted_policy(self, policy: GatewayNotSubmittedPolicy) -> Self {
179        Self {
180            boundary: self.boundary,
181            cooldown: policy.cooldown(),
182        }
183    }
184
185    const fn with_provider(
186        self,
187        provider_key: &GatewayProviderKey,
188    ) -> HostChargeNonApprovedResolution<'_> {
189        HostChargeNonApprovedResolution {
190            boundary: self.boundary,
191            cooldown: match self.cooldown {
192                Some(cooldown) => Some((provider_key, cooldown)),
193                None => None,
194            },
195        }
196    }
197}
198
199pub async fn admit_host_charge_submission(
200    pool: &PgPool,
201    targets: &dyn HostChargeTargetStore,
202    reservation: &HostChargeReservation,
203) -> Result<HostChargeAdmissionOutcome, HostChargeApplicationError> {
204    let mut transaction = pool.begin().await?;
205    let outcome =
206        admit_host_charge_submission_in_transaction(&mut transaction, targets, reservation).await?;
207    transaction.commit().await?;
208    Ok(match outcome {
209        HostChargeSubmissionOutcome::Admitted(attempt) => {
210            HostChargeAdmissionOutcome::Admitted(Box::new(AdmittedHostCharge {
211                reservation: reservation.clone(),
212                attempt,
213            }))
214        }
215        HostChargeSubmissionOutcome::AlreadyAdmitted(attempt) => {
216            HostChargeAdmissionOutcome::AlreadyAdmitted(attempt)
217        }
218        HostChargeSubmissionOutcome::Rejected { attempt, reason } => {
219            HostChargeAdmissionOutcome::Rejected { attempt, reason }
220        }
221    })
222}
223
224pub async fn submit_admitted_host_charge(
225    pool: &PgPool,
226    coordinator: &dyn BillingTransactionCoordinator,
227    targets: &dyn HostChargeTargetStore,
228    admission: AdmittedHostCharge,
229    command: &ChargeHostTarget,
230    gateway: ModeVerifiedGateway<'_>,
231) -> Result<HostChargeProviderResult, HostChargeApplicationError> {
232    let resolved_gateway = gateway.resolved_gateway();
233    let reconstructed = HostChargeReservation::from_command(
234        command,
235        admission.reservation.snapshot(),
236        resolved_gateway,
237        admission.attempt.identity().attempt_id(),
238        admission
239            .reservation
240            .identity()
241            .required_gateway_account_mode(),
242    )
243    .map_err(|_| HostChargeApplicationError::SubmissionIdentityMismatch)?;
244    if reconstructed != admission.reservation
245        || admission.attempt.identity() != admission.reservation.identity()
246        || admission.attempt.request() != admission.reservation.request()
247        || admission.attempt.status() != PaymentAttemptStatus::Pending
248        || admission
249            .attempt
250            .state()
251            .timestamps()
252            .submitted_at()
253            .is_none()
254    {
255        return Err(HostChargeApplicationError::SubmissionIdentityMismatch);
256    }
257    let Some(gateway) = gateway.authorize_attempt(&admission.reservation.identity()) else {
258        return Err(HostChargeApplicationError::SubmissionIdentityMismatch);
259    };
260    let request = GatewaySaleRequest::new(
261        admission.reservation.snapshot().charge(),
262        admission.attempt.request().gateway_order_id().clone(),
263        GatewaySaleIntent::OneTime {
264            payment_token: command.payment_token().clone(),
265        },
266        command.billing_contact().cloned(),
267    );
268    let provider_key = gateway.provider_key().clone();
269    match gateway.sale(request).await {
270        Ok(outcome) => apply_host_charge_gateway_outcome(
271            pool,
272            coordinator,
273            targets,
274            &admission.reservation,
275            &outcome,
276        )
277        .await
278        .map(HostChargeProviderResult::Payment),
279        Err(GatewayMutationError::NotSubmitted(error)) => {
280            let evidence = mutation_error_evidence(error.detail());
281            let policy = GatewayNotSubmittedPolicy::for_error(&error);
282            let application = if policy.restores_prepared_attempt_when_supported() {
283                restore_admitted_host_charge_for_retry(pool, &admission.reservation).await?
284            } else {
285                resolve_host_charge_non_approved(
286                    pool,
287                    targets,
288                    &admission.reservation,
289                    &evidence,
290                    AttemptResolutionStatus::Failed,
291                    Some(policy.resolution_code()),
292                    HostChargeBeforeSubmissionResolution::admitted_not_submitted()
293                        .with_not_submitted_policy(policy)
294                        .with_provider(&provider_key),
295                )
296                .await?
297            };
298            if application.should_surface_not_submitted(policy) {
299                Ok(HostChargeProviderResult::NotSubmitted {
300                    payment: application.payment,
301                    error,
302                })
303            } else {
304                Ok(HostChargeProviderResult::Payment(application.payment))
305            }
306        }
307        Err(GatewayMutationError::RateLimitedIndeterminate(detail)) => resolve_host_charge_unknown(
308            pool,
309            &admission.reservation,
310            &mutation_error_evidence(&detail),
311            Some((&provider_key, RateLimitCooldown::Provider)),
312        )
313        .await
314        .map(HostChargeProviderResult::Payment),
315        Err(GatewayMutationError::Indeterminate(detail)) => resolve_host_charge_unknown(
316            pool,
317            &admission.reservation,
318            &mutation_error_evidence(&detail),
319            None,
320        )
321        .await
322        .map(HostChargeProviderResult::Payment),
323    }
324}
325
326pub async fn apply_host_charge_gateway_outcome(
327    pool: &PgPool,
328    coordinator: &dyn BillingTransactionCoordinator,
329    targets: &dyn HostChargeTargetStore,
330    reservation: &HostChargeReservation,
331    outcome: &GatewayPaymentOutcome,
332) -> Result<HostChargePaymentResult, HostChargeApplicationError> {
333    apply_host_charge_gateway_decision(pool, coordinator, targets, reservation, outcome)
334        .await
335        .map(|result| result.with_gateway_diagnostics(outcome.diagnostics().to_vec()))
336}
337
338async fn apply_host_charge_gateway_decision(
339    pool: &PgPool,
340    coordinator: &dyn BillingTransactionCoordinator,
341    targets: &dyn HostChargeTargetStore,
342    reservation: &HostChargeReservation,
343    outcome: &GatewayPaymentOutcome,
344) -> Result<HostChargePaymentResult, HostChargeApplicationError> {
345    match outcome.status() {
346        GatewayPaymentStatus::Approved => {
347            let approved_evidence =
348                outcome
349                    .approved_evidence()
350                    .ok_or(HostChargeApplicationError::InvalidState(
351                        INVALID_HOST_CHARGE_STATE,
352                    ))?;
353            if outcome.transaction_id().is_none() {
354                return durably_park_host_charge_approved(pool, reservation, &approved_evidence)
355                    .await;
356            }
357            for attempt_index in 0..APPROVED_APPLICATION_ATTEMPTS {
358                match apply_host_charge_approved(
359                    coordinator,
360                    targets,
361                    reservation,
362                    &approved_evidence,
363                )
364                .await
365                {
366                    Ok(result) => return Ok(result),
367                    Err(_) if attempt_index + 1 < APPROVED_APPLICATION_ATTEMPTS => {
368                        tokio::time::sleep(APPROVED_EVIDENCE_RETRY_DELAY).await;
369                    }
370                    Err(_) => break,
371                }
372            }
373            durably_park_host_charge_approved(pool, reservation, &approved_evidence).await
374        }
375        GatewayPaymentStatus::Declined => resolve_host_charge_non_approved(
376            pool,
377            targets,
378            reservation,
379            outcome.evidence(),
380            AttemptResolutionStatus::Declined,
381            None,
382            HostChargeNonApprovedResolution::submitted(),
383        )
384        .await
385        .map(HostChargeResolutionApplication::into_payment),
386        GatewayPaymentStatus::Failed => resolve_host_charge_non_approved(
387            pool,
388            targets,
389            reservation,
390            outcome.evidence(),
391            AttemptResolutionStatus::Failed,
392            None,
393            HostChargeNonApprovedResolution::submitted(),
394        )
395        .await
396        .map(HostChargeResolutionApplication::into_payment),
397        GatewayPaymentStatus::Unknown => {
398            resolve_host_charge_unknown(pool, reservation, outcome.evidence(), None).await
399        }
400    }
401}
402
403pub async fn apply_reconciled_host_charge_gateway_outcome(
404    pool: &PgPool,
405    coordinator: &dyn BillingTransactionCoordinator,
406    targets: &dyn HostChargeTargetStore,
407    billing_scope_id: syrup_rail::BillingScopeId,
408    attempt_id: syrup_rail::PaymentAttemptId,
409    outcome: &GatewayPaymentOutcome,
410) -> Result<HostChargePaymentResult, HostChargeApplicationError> {
411    let mut transaction = pool.begin().await?;
412    let attempt = crate::find_payment_attempt_by_id_in_transaction(
413        &mut transaction,
414        billing_scope_id,
415        attempt_id,
416    )
417    .await?
418    .ok_or(HostChargeApplicationError::InvalidState(
419        INVALID_HOST_CHARGE_STATE,
420    ))?;
421    transaction.commit().await?;
422    let reservation = HostChargeReservation::from_attempt(&attempt)
423        .map_err(|_| HostChargeApplicationError::InvalidState(INVALID_HOST_CHARGE_STATE))?;
424    apply_host_charge_gateway_outcome(pool, coordinator, targets, &reservation, outcome).await
425}
426
427pub(crate) async fn resolve_host_charge_before_submission(
428    pool: &PgPool,
429    targets: &dyn HostChargeTargetStore,
430    reservation: &HostChargeReservation,
431    provider_key: &GatewayProviderKey,
432    detail: syrup_rail::GatewayDiagnostic,
433    resolution_code: PaymentResolutionCode,
434    resolution: HostChargeBeforeSubmissionResolution,
435) -> Result<HostChargePaymentResult, HostChargeApplicationError> {
436    let condition = if matches!(
437        resolution_code,
438        PaymentResolutionCode::GatewayAccountMutationCooldownBeforeSubmission
439            | PaymentResolutionCode::GatewayAccountRateLimitedBeforeSubmission
440            | PaymentResolutionCode::GatewayProviderRateLimitedBeforeSubmission
441    ) {
442        None
443    } else {
444        Some(syrup_rail::GatewayDiagnostic::new("failed"))
445    };
446    let evidence = ProcessorEvidence::new(
447        None,
448        None,
449        None,
450        None,
451        Some(detail),
452        condition,
453        syrup_rail::GatewayPaymentDescriptor::default(),
454    );
455    resolve_host_charge_non_approved(
456        pool,
457        targets,
458        reservation,
459        &evidence,
460        AttemptResolutionStatus::Failed,
461        Some(resolution_code),
462        resolution.with_provider(provider_key),
463    )
464    .await
465    .map(HostChargeResolutionApplication::into_payment)
466}
467
468async fn apply_host_charge_approved(
469    coordinator: &dyn BillingTransactionCoordinator,
470    targets: &dyn HostChargeTargetStore,
471    reservation: &HostChargeReservation,
472    approved_evidence: &ApprovedProcessorEvidence,
473) -> Result<HostChargePaymentResult, HostChargeApplicationError> {
474    let evidence = approved_evidence.evidence();
475    let identity = reservation.identity();
476    let mut transaction = coordinator
477        .begin(
478            BillingEventSubject::new(identity.billing_scope_id(), identity.subscriber_id()),
479            BILLING_LOCK_TIMEOUT,
480        )
481        .await?;
482    if transaction.subject_state() != BillingTransactionSubjectState::LiveRecipient {
483        let _ = transaction.rollback().await;
484        return Err(HostChargeApplicationError::InvalidState(
485            "a host charge payment event requires a live recipient",
486        ));
487    }
488    let connection = transaction.connection();
489    set_application_timeouts(connection).await?;
490    let effective_at: DateTime<Utc> = sqlx::query_scalar("SELECT clock_timestamp()")
491        .fetch_one(&mut *connection)
492        .await?;
493    let target_outcome = targets
494        .apply_transition(
495            connection,
496            HostChargeTargetTransition::new(
497                identity.billing_scope_id(),
498                identity.subscriber_id(),
499                identity.attempt_id(),
500                reservation.snapshot().target_id(),
501                HostChargeTargetTransitionKind::Paid,
502                effective_at,
503            ),
504        )
505        .await?;
506    let attempt = lock_expected_host_charge(connection, reservation).await?;
507    if attempt.status() == PaymentAttemptStatus::Approved {
508        // Host target callbacks establish the repository-wide target -> attempt
509        // lock order. Once this lock proves approval already committed, a
510        // refused Paid replay is harmless: lifecycle reconciliation may have
511        // legitimately advanced the target to a reversed state.
512        observe_processor_charge(
513            connection,
514            &attempt,
515            evidence,
516            ProcessorChargeProgression::Applied,
517        )
518        .await?;
519        transaction.commit().await?;
520        return Ok(HostChargePaymentResult::new(attempt)?);
521    }
522    if attempt.status().is_terminal() {
523        transaction.rollback().await?;
524        return observe_terminal_host_charge_approval(
525            coordinator,
526            reservation,
527            &attempt,
528            approved_evidence,
529        )
530        .await;
531    }
532    let observation = observe_processor_charge(
533        connection,
534        &attempt,
535        evidence,
536        ProcessorChargeProgression::Pending,
537    )
538    .await?;
539    let ObservedCharge::Owned(charge) = observation else {
540        let _ = transaction.rollback().await;
541        return Err(HostChargeApplicationError::InvalidState(
542            "the approved gateway transaction belongs to another payment attempt",
543        ));
544    };
545    if charge.role == ProcessorChargeRole::Additional {
546        let _ = transaction.rollback().await;
547        return Err(HostChargeApplicationError::InvalidState(
548            "an additional approved host charge requires external reversal",
549        ));
550    }
551    match target_outcome {
552        HostChargeTargetTransitionOutcome::Applied
553        | HostChargeTargetTransitionOutcome::ExactReplay => {
554            persist_attempt_transition(
555                connection,
556                &attempt,
557                evidence,
558                AttemptTransition::Approved(AttemptApproval::HostCharge),
559            )
560            .await
561            .map_err(map_attempt_transition_error)?;
562            transition_charge(
563                connection,
564                charge.id,
565                ProcessorChargeProgression::Applied,
566                None,
567            )
568            .await?;
569            let applied = find_payment_attempt_by_id_on_connection(
570                connection,
571                identity.billing_scope_id(),
572                identity.attempt_id(),
573            )
574            .await?
575            .ok_or(HostChargeApplicationError::InvalidState(
576                INVALID_HOST_CHARGE_STATE,
577            ))?;
578            let event = BillingEvent::HostChargePaid {
579                attempt_id: identity.attempt_id(),
580                target_id: reservation.snapshot().target_id(),
581                charge: reservation.snapshot().charge(),
582            };
583            transaction.append_event(&event).await?;
584            transaction.commit().await?;
585            Ok(HostChargePaymentResult::new(applied)?)
586        }
587        HostChargeTargetTransitionOutcome::StaleTarget => {
588            transition_charge(
589                connection,
590                charge.id,
591                ProcessorChargeProgression::ExternalReversalRequired,
592                Some(PaymentResolutionCode::HostChargeApprovedStaleState),
593            )
594            .await?;
595            let parked = park_locked_attempt(
596                connection,
597                &attempt,
598                evidence,
599                Some(PaymentResolutionCode::HostChargeApprovedStaleState),
600                APPROVED_STALE_TARGET_TEXT,
601            )
602            .await?;
603            transaction.commit().await?;
604            Ok(HostChargePaymentResult::new(parked)?)
605        }
606        HostChargeTargetTransitionOutcome::Unchanged { .. } => {
607            let _ = transaction.rollback().await;
608            Err(HostChargeApplicationError::InvalidState(
609                INVALID_HOST_CHARGE_STATE,
610            ))
611        }
612    }
613}
614
615async fn observe_terminal_host_charge_approval(
616    coordinator: &dyn BillingTransactionCoordinator,
617    reservation: &HostChargeReservation,
618    terminal_attempt: &PaymentAttempt,
619    approved_evidence: &ApprovedProcessorEvidence,
620) -> Result<HostChargePaymentResult, HostChargeApplicationError> {
621    let evidence = approved_evidence.evidence();
622    let identity = reservation.identity();
623    let mut transaction = coordinator
624        .begin(
625            BillingEventSubject::new(identity.billing_scope_id(), identity.subscriber_id()),
626            BILLING_LOCK_TIMEOUT,
627        )
628        .await?;
629    let connection = transaction.connection();
630    set_application_timeouts(connection).await?;
631    if matches!(
632        observe_processor_charge(
633            connection,
634            terminal_attempt,
635            evidence,
636            ProcessorChargeProgression::Pending,
637        )
638        .await?,
639        ObservedCharge::OwnedByOtherAttempt
640    ) {
641        let _ = transaction.rollback().await;
642        return Err(HostChargeApplicationError::InvalidState(
643            "the approved gateway transaction belongs to another payment attempt",
644        ));
645    }
646    let locked = lock_expected_host_charge(connection, reservation).await?;
647    if !locked.status().is_terminal() || locked.status() == PaymentAttemptStatus::Approved {
648        let _ = transaction.rollback().await;
649        return Err(HostChargeApplicationError::InvalidState(
650            INVALID_HOST_CHARGE_STATE,
651        ));
652    }
653    transaction.commit().await?;
654    Ok(HostChargePaymentResult::confirmation_pending(
655        locked,
656        approved_evidence.clone(),
657    )?)
658}
659
660async fn resolve_host_charge_non_approved(
661    pool: &PgPool,
662    targets: &dyn HostChargeTargetStore,
663    reservation: &HostChargeReservation,
664    evidence: &ProcessorEvidence,
665    status: AttemptResolutionStatus,
666    resolution_code: Option<PaymentResolutionCode>,
667    resolution: HostChargeNonApprovedResolution<'_>,
668) -> Result<HostChargeResolutionApplication, HostChargeApplicationError> {
669    let identity = reservation.identity();
670    // Provider/account backoff is independent of whether this attempt still
671    // owns the host target or can still be resolved. Commit it first so a
672    // concurrent operator/reconciliation resolution, stale target, or later
673    // host callback failure cannot roll the observed throttle back. A crash
674    // after this commit is fail-safe: it may delay work, but same-key replay
675    // can still finish the unresolved attempt.
676    if let Some((provider_key, cooldown)) = resolution.cooldown {
677        commit_host_charge_cooldown(pool, reservation, provider_key, cooldown).await?;
678    }
679    let mut transaction = pool.begin().await?;
680    set_application_timeouts(&mut transaction).await?;
681    let effective_at = sqlx::query_scalar("SELECT clock_timestamp()")
682        .fetch_one(&mut *transaction)
683        .await?;
684    let target_outcome = targets
685        .apply_transition(
686            &mut transaction,
687            HostChargeTargetTransition::new(
688                identity.billing_scope_id(),
689                identity.subscriber_id(),
690                identity.attempt_id(),
691                reservation.snapshot().target_id(),
692                match resolution.boundary {
693                    OutcomeResolutionBoundary::Prepared
694                    | OutcomeResolutionBoundary::AdmittedNotSubmitted => {
695                        HostChargeTargetTransitionKind::ReleasedBeforeSubmission
696                    }
697                    OutcomeResolutionBoundary::Submitted => {
698                        HostChargeTargetTransitionKind::PaymentFailed
699                    }
700                },
701                effective_at,
702            ),
703        )
704        .await?;
705    if !target_outcome.is_applied() {
706        tracing::warn!(
707            target: "syrup_rail::host_charge_target",
708            billing_scope_id = %identity.billing_scope_id().as_uuid(),
709            subscriber_id = %identity.subscriber_id().as_uuid(),
710            attempt_id = %identity.attempt_id().as_uuid(),
711            target_id = %reservation.snapshot().target_id().as_uuid(),
712            boundary = ?resolution.boundary,
713            ?target_outcome,
714            "host target refused a payment outcome; leaving the canonical attempt unresolved"
715        );
716        transaction.rollback().await?;
717        let application = canonical_host_charge_resolution_application(pool, reservation).await?;
718        if !host_charge_attempt_may_resolve(application.payment.attempt(), resolution.boundary) {
719            return Ok(application);
720        }
721        return Err(HostChargeApplicationError::InvalidState(
722            INVALID_HOST_CHARGE_STATE,
723        ));
724    }
725    let attempt = lock_expected_host_charge(&mut transaction, reservation).await?;
726    let may_resolve = host_charge_attempt_may_resolve(&attempt, resolution.boundary);
727    if !may_resolve {
728        transaction.rollback().await?;
729        return canonical_host_charge_resolution_application(pool, reservation).await;
730    }
731    persist_attempt_transition(
732        &mut transaction,
733        &attempt,
734        evidence,
735        AttemptTransition::Resolved {
736            status,
737            resolution_code,
738        },
739    )
740    .await
741    .map_err(map_attempt_transition_error)?;
742    if evidence.indicates_approved_payment() {
743        observe_processor_charge(
744            &mut transaction,
745            &attempt,
746            evidence,
747            ProcessorChargeProgression::Pending,
748        )
749        .await?;
750    }
751    if resolution.boundary == OutcomeResolutionBoundary::AdmittedNotSubmitted {
752        let cleared = sqlx::query(
753            "UPDATE billing_payment_attempts SET submitted_at = NULL, updated_at = clock_timestamp() WHERE id = $1 AND status = $2",
754        )
755        .bind(identity.attempt_id().as_uuid())
756        .bind(status.as_str())
757        .execute(&mut *transaction)
758        .await?;
759        if cleared.rows_affected() != 1 {
760            return Err(HostChargeApplicationError::InvalidState(
761                INVALID_HOST_CHARGE_STATE,
762            ));
763        }
764    }
765    let attempt = find_payment_attempt_by_id_on_connection(
766        &mut transaction,
767        identity.billing_scope_id(),
768        identity.attempt_id(),
769    )
770    .await?
771    .ok_or(HostChargeApplicationError::InvalidState(
772        INVALID_HOST_CHARGE_STATE,
773    ))?;
774    transaction.commit().await?;
775    Ok(HostChargeResolutionApplication {
776        payment: HostChargePaymentResult::new(attempt)?,
777        applied: true,
778    })
779}
780
781fn host_charge_attempt_may_resolve(
782    attempt: &PaymentAttempt,
783    boundary: OutcomeResolutionBoundary,
784) -> bool {
785    attempt.status().is_resolvable()
786        && match boundary {
787            OutcomeResolutionBoundary::Prepared => {
788                attempt.state().timestamps().submitted_at().is_none()
789            }
790            OutcomeResolutionBoundary::AdmittedNotSubmitted => {
791                attempt.state().timestamps().submitted_at().is_some()
792            }
793            OutcomeResolutionBoundary::Submitted => true,
794        }
795}
796
797async fn canonical_host_charge_resolution_application(
798    pool: &PgPool,
799    reservation: &HostChargeReservation,
800) -> Result<HostChargeResolutionApplication, HostChargeApplicationError> {
801    let identity = reservation.identity();
802    let mut reload = pool.begin().await?;
803    let attempt = crate::find_payment_attempt_by_id_in_transaction(
804        &mut reload,
805        identity.billing_scope_id(),
806        identity.attempt_id(),
807    )
808    .await?
809    .ok_or(HostChargeApplicationError::InvalidState(
810        INVALID_HOST_CHARGE_STATE,
811    ))?;
812    reload.commit().await?;
813    Ok(HostChargeResolutionApplication {
814        payment: HostChargePaymentResult::new(attempt)?,
815        applied: false,
816    })
817}
818
819async fn restore_admitted_host_charge_for_retry(
820    pool: &PgPool,
821    reservation: &HostChargeReservation,
822) -> Result<HostChargeResolutionApplication, HostChargeApplicationError> {
823    // Keep this domain wrapper separate from subscriber restoration: host
824    // charges have their own exact lock and result projection. Both wrappers
825    // delegate the atomic submitted-at transition to the shared primitive.
826    let identity = reservation.identity();
827    let mut transaction = pool.begin().await?;
828    set_application_timeouts(&mut transaction).await?;
829    let attempt = lock_expected_host_charge(&mut transaction, reservation).await?;
830    let restored = attempt.status() == PaymentAttemptStatus::Pending
831        && attempt.state().timestamps().submitted_at().is_some();
832    if restored {
833        restore_prepared_attempt_submission(&mut transaction, &attempt).await?;
834    }
835    let attempt = find_payment_attempt_by_id_on_connection(
836        &mut transaction,
837        identity.billing_scope_id(),
838        identity.attempt_id(),
839    )
840    .await?
841    .ok_or(HostChargeApplicationError::InvalidState(
842        INVALID_HOST_CHARGE_STATE,
843    ))?;
844    transaction.commit().await?;
845    if restored {
846        tracing::warn!(
847            target: "syrup_rail::gateway_control_plane",
848            attempt_id = %identity.attempt_id().as_uuid(),
849            attempt_kind = attempt.kind().as_str(),
850            required_gateway_account_mode = identity.required_gateway_account_mode().as_str(),
851            "restored admitted host charge after pre-submission control-plane failure"
852        );
853    }
854    Ok(HostChargeResolutionApplication {
855        payment: HostChargePaymentResult::new(attempt)?,
856        applied: restored,
857    })
858}
859
860async fn resolve_host_charge_unknown(
861    pool: &PgPool,
862    reservation: &HostChargeReservation,
863    evidence: &ProcessorEvidence,
864    cooldown: Option<(&GatewayProviderKey, RateLimitCooldown)>,
865) -> Result<HostChargePaymentResult, HostChargeApplicationError> {
866    let identity = reservation.identity();
867    if let Some((provider_key, cooldown)) = cooldown {
868        commit_host_charge_cooldown(pool, reservation, provider_key, cooldown).await?;
869    }
870    let mut transaction = pool.begin().await?;
871    set_application_timeouts(&mut transaction).await?;
872    let attempt = lock_expected_host_charge(&mut transaction, reservation).await?;
873    if !attempt.status().is_terminal() {
874        persist_attempt_transition(
875            &mut transaction,
876            &attempt,
877            evidence,
878            AttemptTransition::Resolved {
879                status: AttemptResolutionStatus::Unknown,
880                resolution_code: None,
881            },
882        )
883        .await
884        .map_err(map_attempt_transition_error)?;
885        if evidence.indicates_approved_payment() {
886            observe_processor_charge(
887                &mut transaction,
888                &attempt,
889                evidence,
890                ProcessorChargeProgression::Pending,
891            )
892            .await?;
893        }
894    }
895    let attempt = find_payment_attempt_by_id_on_connection(
896        &mut transaction,
897        identity.billing_scope_id(),
898        identity.attempt_id(),
899    )
900    .await?
901    .ok_or(HostChargeApplicationError::InvalidState(
902        INVALID_HOST_CHARGE_STATE,
903    ))?;
904    transaction.commit().await?;
905    Ok(HostChargePaymentResult::new(attempt)?)
906}
907
908async fn commit_host_charge_cooldown(
909    pool: &PgPool,
910    reservation: &HostChargeReservation,
911    provider_key: &GatewayProviderKey,
912    cooldown: RateLimitCooldown,
913) -> Result<(), HostChargeApplicationError> {
914    match commit_rate_limit_cooldown_for_operation(
915        pool,
916        reservation.identity(),
917        provider_key,
918        cooldown,
919        RateLimitCooldownOperation::HostCharge,
920    )
921    .await
922    {
923        Ok(()) => Ok(()),
924        Err(RateLimitCooldownCommitError::Sql(error)) => Err(error.into()),
925        Err(RateLimitCooldownCommitError::MissingProviderCooldown) => Err(
926            HostChargeApplicationError::InvalidState(INVALID_HOST_CHARGE_STATE),
927        ),
928    }
929}
930
931async fn park_host_charge_approved(
932    pool: &PgPool,
933    reservation: &HostChargeReservation,
934    evidence: &ProcessorEvidence,
935) -> Result<HostChargePaymentResult, HostChargeApplicationError> {
936    let mut transaction = pool.begin().await?;
937    set_application_timeouts(&mut transaction).await?;
938    let attempt = lock_expected_host_charge(&mut transaction, reservation).await?;
939    let progression = if evidence.transaction_id().is_some() {
940        ProcessorChargeProgression::ExternalReversalRequired
941    } else {
942        ProcessorChargeProgression::ReconciliationRequired
943    };
944    let observation =
945        observe_processor_charge(&mut transaction, &attempt, evidence, progression).await?;
946    if let ObservedCharge::Owned(charge) = observation {
947        transition_charge(&mut transaction, charge.id, progression, None).await?;
948    }
949    let parked = park_locked_attempt(
950        &mut transaction,
951        &attempt,
952        evidence,
953        None,
954        APPROVED_STORAGE_FAILURE_TEXT,
955    )
956    .await?;
957    transaction.commit().await?;
958    Ok(HostChargePaymentResult::new(parked)?)
959}
960
961async fn durably_park_host_charge_approved(
962    pool: &PgPool,
963    reservation: &HostChargeReservation,
964    approved_evidence: &ApprovedProcessorEvidence,
965) -> Result<HostChargePaymentResult, HostChargeApplicationError> {
966    let evidence = approved_evidence.evidence();
967    if let Ok(payment) = park_host_charge_approved(pool, reservation, evidence).await {
968        return Ok(payment);
969    }
970    crate::store_compensating_processor_charge(
971        pool,
972        reservation.identity().attempt_id(),
973        reservation.request().gateway_order_id(),
974        evidence,
975    )
976    .await?;
977    let mut transaction = pool.begin().await?;
978    let attempt = crate::find_payment_attempt_by_id_in_transaction(
979        &mut transaction,
980        reservation.identity().billing_scope_id(),
981        reservation.identity().attempt_id(),
982    )
983    .await?
984    .ok_or(HostChargeApplicationError::InvalidState(
985        INVALID_HOST_CHARGE_STATE,
986    ))?;
987    transaction.commit().await?;
988    Ok(HostChargePaymentResult::confirmation_pending(
989        attempt,
990        approved_evidence.clone(),
991    )?)
992}
993
994async fn lock_expected_host_charge(
995    connection: &mut PgConnection,
996    reservation: &HostChargeReservation,
997) -> Result<PaymentAttempt, HostChargeApplicationError> {
998    let identity = reservation.identity();
999    let attempt = lock_payment_attempt_by_id_on_connection(
1000        connection,
1001        identity.billing_scope_id(),
1002        identity.attempt_id(),
1003    )
1004    .await?
1005    .ok_or(HostChargeApplicationError::InvalidState(
1006        INVALID_HOST_CHARGE_STATE,
1007    ))?;
1008    if !host_charge_attempt_matches_reservation(&attempt, reservation) {
1009        return Err(HostChargeApplicationError::InvalidState(
1010            INVALID_HOST_CHARGE_STATE,
1011        ));
1012    }
1013    Ok(attempt)
1014}
1015
1016#[cfg(test)]
1017mod tests {
1018    use std::{
1019        error::Error,
1020        sync::{
1021            Arc,
1022            atomic::{AtomicUsize, Ordering},
1023        },
1024    };
1025
1026    use async_trait::async_trait;
1027    use chrono::Duration as ChronoDuration;
1028    use sqlx::{Postgres, Transaction};
1029    use syrup_rail::{
1030        BillingContact, BillingEventKey, ChargeAmount, CurrencyCode, EndUserMutationAdmission,
1031        EndUserMutationAdmissionResult, EndUserMutationCommand, GatewayAccountId,
1032        GatewayAccountMode, GatewayConfigurationId, GatewayDiagnostic, GatewayError,
1033        GatewayLifecycleCursorKey, GatewayLifecycleQueryPolicy, GatewayMutationError,
1034        GatewayMutationReferenceFactory, GatewayOrderId, GatewayPaymentDescriptor,
1035        GatewayPaymentDiagnostic, GatewayProviderKey, GatewayQueryRequest, GatewayResolutionError,
1036        GatewayResolver, GatewayStorePaymentMethodRequest, GatewayTransactionId,
1037        GatewayTransactionReport, GatewayTransactionReportRequest, HostChargeTargetId,
1038        HostChargeTargetNoChange, IdempotencyKey, PaymentAttemptId, PaymentAttemptKind,
1039        PaymentGateway, PaymentToken, ResolvedGateway,
1040    };
1041    use tokio::sync::{Mutex, Notify, oneshot};
1042    use uuid::Uuid;
1043
1044    use super::*;
1045    use crate::{
1046        BillingEventWriteError, BillingTransaction, HostChargeLedgerAdmission,
1047        HostChargeLedgerAdmissionMode, HostChargeLedgerAdmissionQuery,
1048        HostChargeReservationDecision, HostChargeReservationOutcome, HostChargeSubmissionAdmission,
1049        HostChargeSubmissionDecision, HostChargeTargetReservation, SubscriptionBillingService,
1050        SubscriptionOfferStore, host_charge_ledger_admission, reserve_host_charge_in_transaction,
1051        test_support::{TestDatabase, create_gateway_account},
1052    };
1053
1054    mod readiness_replay;
1055
1056    #[test]
1057    fn before_submission_resolution_modes_keep_boundary_and_cooldown_distinct() {
1058        let prepared = HostChargeBeforeSubmissionResolution::prepared();
1059        assert_eq!(prepared.boundary, OutcomeResolutionBoundary::Prepared);
1060        assert!(prepared.cooldown.is_none());
1061
1062        let admitted = HostChargeBeforeSubmissionResolution::admitted_not_submitted();
1063        assert_eq!(
1064            admitted.boundary,
1065            OutcomeResolutionBoundary::AdmittedNotSubmitted
1066        );
1067        assert!(admitted.cooldown.is_none());
1068
1069        let rate_limited = HostChargeBeforeSubmissionResolution::prepared()
1070            .with_not_submitted_policy(GatewayNotSubmittedPolicy::for_readiness_error(
1071                &syrup_rail::GatewayError::RateLimited(GatewayDiagnostic::new("rate limited")),
1072            ));
1073        assert_eq!(rate_limited.boundary, OutcomeResolutionBoundary::Prepared);
1074        assert!(matches!(
1075            rate_limited.cooldown,
1076            Some(RateLimitCooldown::Provider)
1077        ));
1078
1079        let terminal_failure = HostChargeBeforeSubmissionResolution::prepared()
1080            .with_not_submitted_policy(GatewayNotSubmittedPolicy::for_readiness_error(
1081                &syrup_rail::GatewayError::Configuration(GatewayDiagnostic::new("configuration")),
1082            ));
1083        assert_eq!(
1084            terminal_failure.boundary,
1085            OutcomeResolutionBoundary::Prepared
1086        );
1087        assert!(terminal_failure.cooldown.is_none());
1088    }
1089
1090    struct TestReferenceFactory;
1091
1092    impl GatewayMutationReferenceFactory for TestReferenceFactory {
1093        fn for_attempt(
1094            &self,
1095            _kind: PaymentAttemptKind,
1096            attempt_id: PaymentAttemptId,
1097        ) -> GatewayOrderId {
1098            GatewayOrderId::from_generated_attempt(
1099                format!("test_host_{}", attempt_id.as_uuid().simple()),
1100                attempt_id,
1101            )
1102            .expect("valid host test reference")
1103        }
1104    }
1105
1106    struct ScriptedGateway {
1107        account_mode: GatewayAccountMode,
1108        sale_calls: AtomicUsize,
1109        outcome: Mutex<Option<GatewayPaymentOutcome>>,
1110    }
1111
1112    #[async_trait]
1113    impl PaymentGateway for ScriptedGateway {
1114        async fn account_mode(&self) -> Result<GatewayAccountMode, GatewayError> {
1115            Ok(self.account_mode)
1116        }
1117
1118        async fn sale(
1119            &self,
1120            _request: GatewaySaleRequest,
1121        ) -> Result<GatewayPaymentOutcome, GatewayMutationError> {
1122            self.sale_calls.fetch_add(1, Ordering::SeqCst);
1123            Ok(self
1124                .outcome
1125                .lock()
1126                .await
1127                .take()
1128                .expect("one sale capability"))
1129        }
1130
1131        async fn store_payment_method(
1132            &self,
1133            _request: GatewayStorePaymentMethodRequest,
1134        ) -> Result<GatewayPaymentOutcome, GatewayMutationError> {
1135            panic!("host charge must not store a payment method")
1136        }
1137
1138        async fn query_transaction(
1139            &self,
1140            _request: GatewayQueryRequest,
1141        ) -> Result<Option<GatewayPaymentOutcome>, GatewayError> {
1142            panic!("foreground host charge must not query")
1143        }
1144
1145        async fn query_transaction_reports(
1146            &self,
1147            _request: GatewayTransactionReportRequest,
1148        ) -> Result<Vec<GatewayTransactionReport>, GatewayError> {
1149            panic!("foreground host charge must not query reports")
1150        }
1151    }
1152
1153    struct RateLimitedAfterReservationGateway {
1154        readiness_calls: AtomicUsize,
1155        sale_calls: AtomicUsize,
1156    }
1157
1158    struct TerminalRaceGateway {
1159        pool: PgPool,
1160        sale_calls: AtomicUsize,
1161    }
1162
1163    struct RacingPreparedRetryGateway {
1164        readiness_calls: AtomicUsize,
1165        sale_calls: AtomicUsize,
1166        blocked_readiness_started: Mutex<Option<oneshot::Sender<()>>>,
1167        sale_started: Mutex<Option<oneshot::Sender<()>>>,
1168        release_readiness: Notify,
1169        release_sale: Notify,
1170    }
1171
1172    #[async_trait]
1173    impl PaymentGateway for RacingPreparedRetryGateway {
1174        async fn account_mode(&self) -> Result<GatewayAccountMode, GatewayError> {
1175            if self.readiness_calls.fetch_add(1, Ordering::SeqCst) == 0 {
1176                if let Some(started) = self.blocked_readiness_started.lock().await.take() {
1177                    let _ = started.send(());
1178                }
1179                self.release_readiness.notified().await;
1180                Ok(GatewayAccountMode::Test)
1181            } else {
1182                Ok(GatewayAccountMode::Live)
1183            }
1184        }
1185
1186        async fn sale(
1187            &self,
1188            _request: GatewaySaleRequest,
1189        ) -> Result<GatewayPaymentOutcome, GatewayMutationError> {
1190            self.sale_calls.fetch_add(1, Ordering::SeqCst);
1191            if let Some(started) = self.sale_started.lock().await.take() {
1192                let _ = started.send(());
1193            }
1194            self.release_sale.notified().await;
1195            Ok(approved_outcome("host_txn_prepared_retry"))
1196        }
1197
1198        async fn store_payment_method(
1199            &self,
1200            _request: GatewayStorePaymentMethodRequest,
1201        ) -> Result<GatewayPaymentOutcome, GatewayMutationError> {
1202            panic!("host charge must not store a payment method")
1203        }
1204
1205        async fn query_transaction(
1206            &self,
1207            _request: GatewayQueryRequest,
1208        ) -> Result<Option<GatewayPaymentOutcome>, GatewayError> {
1209            panic!("foreground host charge must not query")
1210        }
1211
1212        async fn query_transaction_reports(
1213            &self,
1214            _request: GatewayTransactionReportRequest,
1215        ) -> Result<Vec<GatewayTransactionReport>, GatewayError> {
1216            panic!("foreground host charge must not query reports")
1217        }
1218    }
1219
1220    #[async_trait]
1221    impl PaymentGateway for TerminalRaceGateway {
1222        async fn account_mode(&self) -> Result<GatewayAccountMode, GatewayError> {
1223            Ok(GatewayAccountMode::Live)
1224        }
1225
1226        async fn sale(
1227            &self,
1228            request: GatewaySaleRequest,
1229        ) -> Result<GatewayPaymentOutcome, GatewayMutationError> {
1230            self.sale_calls.fetch_add(1, Ordering::SeqCst);
1231            let updated = sqlx::query(
1232                r#"
1233                UPDATE billing_payment_attempts
1234                SET status = 'failed', resolved_at = clock_timestamp(),
1235                    gateway_response_text = 'simulated terminal race',
1236                    gateway_condition = 'failed', updated_at = clock_timestamp()
1237                WHERE gateway_order_id = $1 AND status = 'pending'
1238                "#,
1239            )
1240            .bind(request.order_id().expose())
1241            .execute(&self.pool)
1242            .await
1243            .expect("simulate a terminal attempt race");
1244            assert_eq!(updated.rows_affected(), 1);
1245            Ok(approved_outcome("host_txn_terminal_race"))
1246        }
1247
1248        async fn store_payment_method(
1249            &self,
1250            _request: GatewayStorePaymentMethodRequest,
1251        ) -> Result<GatewayPaymentOutcome, GatewayMutationError> {
1252            panic!("host charge must not store a payment method")
1253        }
1254
1255        async fn query_transaction(
1256            &self,
1257            _request: GatewayQueryRequest,
1258        ) -> Result<Option<GatewayPaymentOutcome>, GatewayError> {
1259            panic!("foreground host charge must not query")
1260        }
1261
1262        async fn query_transaction_reports(
1263            &self,
1264            _request: GatewayTransactionReportRequest,
1265        ) -> Result<Vec<GatewayTransactionReport>, GatewayError> {
1266            panic!("foreground host charge must not query reports")
1267        }
1268    }
1269
1270    #[async_trait]
1271    impl PaymentGateway for RateLimitedAfterReservationGateway {
1272        async fn account_mode(&self) -> Result<GatewayAccountMode, GatewayError> {
1273            self.readiness_calls.fetch_add(1, Ordering::SeqCst);
1274            Err(GatewayError::RateLimited(GatewayDiagnostic::new(
1275                "provider throttled the readiness check",
1276            )))
1277        }
1278
1279        async fn sale(
1280            &self,
1281            _request: GatewaySaleRequest,
1282        ) -> Result<GatewayPaymentOutcome, GatewayMutationError> {
1283            self.sale_calls.fetch_add(1, Ordering::SeqCst);
1284            panic!("rate-limited host charge must not submit a sale")
1285        }
1286
1287        async fn store_payment_method(
1288            &self,
1289            _request: GatewayStorePaymentMethodRequest,
1290        ) -> Result<GatewayPaymentOutcome, GatewayMutationError> {
1291            panic!("host charge must not store a payment method")
1292        }
1293
1294        async fn query_transaction(
1295            &self,
1296            _request: GatewayQueryRequest,
1297        ) -> Result<Option<GatewayPaymentOutcome>, GatewayError> {
1298            panic!("foreground host charge must not query")
1299        }
1300
1301        async fn query_transaction_reports(
1302            &self,
1303            _request: GatewayTransactionReportRequest,
1304        ) -> Result<Vec<GatewayTransactionReport>, GatewayError> {
1305            panic!("foreground host charge must not query reports")
1306        }
1307    }
1308
1309    struct StaticResolver {
1310        gateway: ResolvedGateway,
1311        calls: AtomicUsize,
1312    }
1313
1314    #[async_trait]
1315    impl GatewayResolver for StaticResolver {
1316        async fn resolve(
1317            &self,
1318            billing_scope_id: syrup_rail::BillingScopeId,
1319            gateway_account_id: GatewayAccountId,
1320            gateway_configuration_id: GatewayConfigurationId,
1321            provider_key: GatewayProviderKey,
1322        ) -> Result<ResolvedGateway, GatewayResolutionError> {
1323            self.calls.fetch_add(1, Ordering::SeqCst);
1324            if billing_scope_id != self.gateway.billing_scope_id()
1325                || gateway_account_id != self.gateway.gateway_account_id()
1326                || gateway_configuration_id != self.gateway.gateway_configuration_id()
1327                || provider_key != *self.gateway.provider_key()
1328            {
1329                return Err(GatewayResolutionError::ConfigurationChanged);
1330            }
1331            Ok(self.gateway.clone())
1332        }
1333    }
1334
1335    struct PermitAdmission {
1336        calls: AtomicUsize,
1337    }
1338
1339    #[async_trait]
1340    impl EndUserMutationAdmission for PermitAdmission {
1341        async fn admit(&self, _command: EndUserMutationCommand) -> EndUserMutationAdmissionResult {
1342            self.calls.fetch_add(1, Ordering::SeqCst);
1343            EndUserMutationAdmissionResult::Allowed
1344        }
1345    }
1346
1347    struct UnusedOffers;
1348
1349    #[async_trait]
1350    impl SubscriptionOfferStore for UnusedOffers {
1351        async fn lock_current_offer(
1352            &self,
1353            _connection: &mut PgConnection,
1354            _billing_scope_id: syrup_rail::BillingScopeId,
1355            _plan_key: &syrup_rail::PlanKey,
1356        ) -> Result<Option<syrup_rail::SubscriptionOffer>, sqlx::Error> {
1357            panic!("host charge must not load a subscription offer")
1358        }
1359    }
1360
1361    #[derive(Clone)]
1362    struct TestCoordinator {
1363        pool: PgPool,
1364        events: Arc<Mutex<Vec<BillingEvent>>>,
1365    }
1366
1367    #[async_trait]
1368    impl BillingTransactionCoordinator for TestCoordinator {
1369        async fn begin(
1370            &self,
1371            _subject: BillingEventSubject,
1372            _lock_timeout: Duration,
1373        ) -> Result<Box<dyn BillingTransaction>, BillingTransactionError> {
1374            Ok(Box::new(TestTransaction {
1375                transaction: Some(
1376                    self.pool
1377                        .begin()
1378                        .await
1379                        .map_err(BillingTransactionError::new)?,
1380                ),
1381                events: Arc::clone(&self.events),
1382            }))
1383        }
1384    }
1385
1386    struct TestTransaction {
1387        transaction: Option<Transaction<'static, Postgres>>,
1388        events: Arc<Mutex<Vec<BillingEvent>>>,
1389    }
1390
1391    #[async_trait]
1392    impl BillingTransaction for TestTransaction {
1393        fn connection(&mut self) -> &mut PgConnection {
1394            &mut *self.transaction.as_mut().expect("active transaction")
1395        }
1396
1397        fn subject_state(&self) -> BillingTransactionSubjectState {
1398            BillingTransactionSubjectState::LiveRecipient
1399        }
1400
1401        async fn append_event(
1402            &mut self,
1403            event: &BillingEvent,
1404        ) -> Result<(), BillingEventWriteError> {
1405            self.events.lock().await.push(event.clone());
1406            Ok(())
1407        }
1408
1409        async fn commit(mut self: Box<Self>) -> Result<(), BillingTransactionError> {
1410            self.transaction
1411                .take()
1412                .expect("active transaction")
1413                .commit()
1414                .await
1415                .map_err(BillingTransactionError::new)
1416        }
1417
1418        async fn rollback(mut self: Box<Self>) -> Result<(), BillingTransactionError> {
1419            self.transaction
1420                .take()
1421                .expect("active transaction")
1422                .rollback()
1423                .await
1424                .map_err(BillingTransactionError::new)
1425        }
1426    }
1427
1428    struct TestTargets;
1429
1430    struct AdmissionMustNotRun;
1431
1432    #[async_trait]
1433    impl HostChargeTargetStore for AdmissionMustNotRun {
1434        async fn preflight_target(
1435            &self,
1436            _connection: &mut PgConnection,
1437            _reservation: &HostChargeTargetReservation,
1438        ) -> Result<HostChargeReservationDecision, HostChargeTargetError> {
1439            panic!("terminal admission replay must not preflight the host target")
1440        }
1441
1442        async fn reserve_target(
1443            &self,
1444            _connection: &mut PgConnection,
1445            _reservation: &HostChargeTargetReservation,
1446        ) -> Result<HostChargeReservationDecision, HostChargeTargetError> {
1447            panic!("terminal admission replay must not reserve the host target")
1448        }
1449
1450        async fn ensure_submission_admitted(
1451            &self,
1452            _connection: &mut PgConnection,
1453            _admission: &HostChargeSubmissionAdmission,
1454        ) -> Result<HostChargeSubmissionDecision, HostChargeTargetError> {
1455            panic!("terminal admission replay must not invoke the host target")
1456        }
1457
1458        async fn apply_transition(
1459            &self,
1460            _connection: &mut PgConnection,
1461            _transition: HostChargeTargetTransition,
1462        ) -> Result<HostChargeTargetTransitionOutcome, HostChargeTargetError> {
1463            panic!("terminal admission replay must not transition the host target")
1464        }
1465    }
1466
1467    #[async_trait]
1468    impl HostChargeTargetStore for TestTargets {
1469        async fn preflight_target(
1470            &self,
1471            connection: &mut PgConnection,
1472            reservation: &HostChargeTargetReservation,
1473        ) -> Result<HostChargeReservationDecision, HostChargeTargetError> {
1474            self.reserve_target(connection, reservation).await
1475        }
1476
1477        async fn reserve_target(
1478            &self,
1479            connection: &mut PgConnection,
1480            reservation: &HostChargeTargetReservation,
1481        ) -> Result<HostChargeReservationDecision, HostChargeTargetError> {
1482            let row = sqlx::query_as::<_, (String, i32, String)>(
1483                r#"
1484                SELECT status, amount_cents, currency
1485                FROM host_charge_targets
1486                WHERE id = $1 AND billing_scope_id = $2 AND subscriber_id = $3
1487                FOR UPDATE
1488                "#,
1489            )
1490            .bind(reservation.target_id().as_uuid())
1491            .bind(reservation.billing_scope_id().as_uuid())
1492            .bind(reservation.subscriber_id().as_uuid())
1493            .fetch_optional(&mut *connection)
1494            .await
1495            .map_err(HostChargeTargetError::new)?;
1496            let Some((status, cents, currency)) = row else {
1497                return Ok(HostChargeReservationDecision::Rejected {
1498                    reason: syrup_rail::HostChargeTargetRejection::TargetUnavailable,
1499                });
1500            };
1501            let ledger = host_charge_ledger_admission(
1502                connection,
1503                &HostChargeLedgerAdmissionQuery::new(
1504                    reservation.billing_scope_id(),
1505                    reservation.subscriber_id(),
1506                    reservation.target_id(),
1507                    HostChargeLedgerAdmissionMode::Reserve {
1508                        idempotency_key: reservation.idempotency_key().clone(),
1509                    },
1510                ),
1511            )
1512            .await
1513            .map_err(HostChargeTargetError::new)?;
1514            let charge = ChargeAmount::new(cents, CurrencyCode::new(&currency).unwrap()).unwrap();
1515            if ledger == HostChargeLedgerAdmission::IdempotentContender {
1516                return Ok(HostChargeReservationDecision::IdempotentContender(
1517                    syrup_rail::HostChargeTargetSnapshot::new(reservation.target_id(), charge),
1518                ));
1519            }
1520            if ledger != HostChargeLedgerAdmission::Safe || status != "pending" {
1521                return Ok(HostChargeReservationDecision::Rejected {
1522                    reason: syrup_rail::HostChargeTargetRejection::LedgerUnsafe,
1523                });
1524            }
1525            Ok(HostChargeReservationDecision::Reserved(
1526                syrup_rail::HostChargeTargetSnapshot::new(reservation.target_id(), charge),
1527            ))
1528        }
1529
1530        async fn ensure_submission_admitted(
1531            &self,
1532            connection: &mut PgConnection,
1533            admission: &HostChargeSubmissionAdmission,
1534        ) -> Result<HostChargeSubmissionDecision, HostChargeTargetError> {
1535            let row = sqlx::query_as::<_, (String, i32, String)>(
1536                r#"
1537                SELECT status, amount_cents, currency
1538                FROM host_charge_targets
1539                WHERE id = $1 AND billing_scope_id = $2 AND subscriber_id = $3
1540                FOR UPDATE
1541                "#,
1542            )
1543            .bind(admission.target_id().as_uuid())
1544            .bind(admission.billing_scope_id().as_uuid())
1545            .bind(admission.subscriber_id().as_uuid())
1546            .fetch_optional(&mut *connection)
1547            .await
1548            .map_err(HostChargeTargetError::new)?;
1549            let Some((status, cents, currency)) = row else {
1550                return Ok(HostChargeSubmissionDecision::Rejected {
1551                    reason: syrup_rail::HostChargeTargetRejection::TargetUnavailable,
1552                });
1553            };
1554            let charge = ChargeAmount::new(cents, CurrencyCode::new(&currency).unwrap()).unwrap();
1555            let ledger = host_charge_ledger_admission(
1556                connection,
1557                &HostChargeLedgerAdmissionQuery::new(
1558                    admission.billing_scope_id(),
1559                    admission.subscriber_id(),
1560                    admission.target_id(),
1561                    HostChargeLedgerAdmissionMode::Submit {
1562                        attempt_id: admission.attempt_id(),
1563                    },
1564                ),
1565            )
1566            .await
1567            .map_err(HostChargeTargetError::new)?;
1568            if ledger == HostChargeLedgerAdmission::Safe
1569                && status == "pending"
1570                && charge == admission.expected_charge()
1571            {
1572                Ok(HostChargeSubmissionDecision::Admitted(
1573                    syrup_rail::HostChargeTargetSnapshot::new(admission.target_id(), charge),
1574                ))
1575            } else {
1576                Ok(HostChargeSubmissionDecision::Rejected {
1577                    reason: syrup_rail::HostChargeTargetRejection::ChargeChanged,
1578                })
1579            }
1580        }
1581
1582        async fn apply_transition(
1583            &self,
1584            connection: &mut PgConnection,
1585            transition: HostChargeTargetTransition,
1586        ) -> Result<HostChargeTargetTransitionOutcome, HostChargeTargetError> {
1587            let current: Option<String> = sqlx::query_scalar(
1588                r#"
1589                SELECT status FROM host_charge_targets
1590                WHERE id = $1 AND billing_scope_id = $2 AND subscriber_id = $3
1591                FOR UPDATE
1592                "#,
1593            )
1594            .bind(transition.target_id().as_uuid())
1595            .bind(transition.billing_scope_id().as_uuid())
1596            .bind(transition.subscriber_id().as_uuid())
1597            .fetch_optional(&mut *connection)
1598            .await
1599            .map_err(HostChargeTargetError::new)?;
1600            let Some(current) = current else {
1601                return Ok(HostChargeTargetTransitionOutcome::StaleTarget);
1602            };
1603            match transition.kind() {
1604                HostChargeTargetTransitionKind::Paid if current == "pending" => {
1605                    sqlx::query(
1606                        "UPDATE host_charge_targets SET status = 'paid', paid_at = $2 WHERE id = $1",
1607                    )
1608                    .bind(transition.target_id().as_uuid())
1609                    .bind(transition.effective_at())
1610                    .execute(connection)
1611                    .await
1612                    .map_err(HostChargeTargetError::new)?;
1613                    Ok(HostChargeTargetTransitionOutcome::Applied)
1614                }
1615                HostChargeTargetTransitionKind::Paid if current == "paid" => {
1616                    Ok(HostChargeTargetTransitionOutcome::ExactReplay)
1617                }
1618                HostChargeTargetTransitionKind::Paid => {
1619                    Ok(HostChargeTargetTransitionOutcome::StaleTarget)
1620                }
1621                HostChargeTargetTransitionKind::ReleasedBeforeSubmission
1622                | HostChargeTargetTransitionKind::PaymentFailed
1623                    if current == "pending" =>
1624                {
1625                    Ok(HostChargeTargetTransitionOutcome::Applied)
1626                }
1627                _ => Ok(HostChargeTargetTransitionOutcome::Unchanged {
1628                    reason: HostChargeTargetNoChange::InapplicableState,
1629                }),
1630            }
1631        }
1632    }
1633
1634    struct RefusingTransitionTargets;
1635
1636    #[async_trait]
1637    impl HostChargeTargetStore for RefusingTransitionTargets {
1638        async fn preflight_target(
1639            &self,
1640            connection: &mut PgConnection,
1641            reservation: &HostChargeTargetReservation,
1642        ) -> Result<HostChargeReservationDecision, HostChargeTargetError> {
1643            TestTargets.preflight_target(connection, reservation).await
1644        }
1645
1646        async fn reserve_target(
1647            &self,
1648            connection: &mut PgConnection,
1649            reservation: &HostChargeTargetReservation,
1650        ) -> Result<HostChargeReservationDecision, HostChargeTargetError> {
1651            TestTargets.reserve_target(connection, reservation).await
1652        }
1653
1654        async fn ensure_submission_admitted(
1655            &self,
1656            connection: &mut PgConnection,
1657            admission: &HostChargeSubmissionAdmission,
1658        ) -> Result<HostChargeSubmissionDecision, HostChargeTargetError> {
1659            TestTargets
1660                .ensure_submission_admitted(connection, admission)
1661                .await
1662        }
1663
1664        async fn apply_transition(
1665            &self,
1666            _connection: &mut PgConnection,
1667            _transition: HostChargeTargetTransition,
1668        ) -> Result<HostChargeTargetTransitionOutcome, HostChargeTargetError> {
1669            Ok(HostChargeTargetTransitionOutcome::Unchanged {
1670                reason: HostChargeTargetNoChange::ReleaseUnsafe,
1671            })
1672        }
1673    }
1674
1675    fn resolved_gateway(
1676        account: crate::test_support::GatewayAccountFixture,
1677        gateway: Arc<dyn PaymentGateway>,
1678    ) -> ResolvedGateway {
1679        ResolvedGateway::new(
1680            syrup_rail::BillingScopeId::new(account.billing_scope_id),
1681            GatewayAccountId::new(account.gateway_account_id),
1682            GatewayConfigurationId::new(account.gateway_configuration_id),
1683            GatewayProviderKey::new("nmi").unwrap(),
1684            GatewayLifecycleQueryPolicy::new(
1685                GatewayLifecycleCursorKey::new("host_test").unwrap(),
1686                ChronoDuration::minutes(1),
1687                10,
1688                2,
1689                2,
1690                20,
1691            )
1692            .unwrap(),
1693            Arc::new(TestReferenceFactory),
1694            gateway,
1695        )
1696    }
1697
1698    fn approved_outcome(transaction_id: &str) -> GatewayPaymentOutcome {
1699        GatewayPaymentOutcome::new(
1700            GatewayPaymentStatus::Approved,
1701            ProcessorEvidence::new(
1702                Some(GatewayTransactionId::new(transaction_id).unwrap()),
1703                None,
1704                Some(GatewayDiagnostic::new("1")),
1705                None,
1706                Some(GatewayDiagnostic::new("approved")),
1707                Some(GatewayDiagnostic::new("complete")),
1708                GatewayPaymentDescriptor::default(),
1709            ),
1710        )
1711    }
1712
1713    #[tokio::test]
1714    async fn foreground_host_charge_is_one_shot_atomic_and_replay_first()
1715    -> Result<(), Box<dyn Error>> {
1716        let database = TestDatabase::start("rail_host_svc").await?;
1717        let result = async {
1718            sqlx::query(
1719                r#"
1720                CREATE TABLE host_charge_targets (
1721                    id uuid PRIMARY KEY,
1722                    billing_scope_id uuid NOT NULL,
1723                    subscriber_id uuid NOT NULL,
1724                    status text NOT NULL,
1725                    amount_cents integer NOT NULL,
1726                    currency text NOT NULL,
1727                    paid_at timestamptz
1728                )
1729                "#,
1730            )
1731            .execute(&database.pool)
1732            .await?;
1733            let account = create_gateway_account(&database.pool, "nmi").await?;
1734            let subscriber_id = Uuid::now_v7();
1735            let target_id = Uuid::now_v7();
1736            sqlx::query(
1737                "INSERT INTO host_charge_targets VALUES ($1, $2, $3, 'pending', 1250, 'USD', NULL)",
1738            )
1739            .bind(target_id)
1740            .bind(account.billing_scope_id)
1741            .bind(subscriber_id)
1742            .execute(&database.pool)
1743            .await?;
1744
1745            let gateway = Arc::new(ScriptedGateway {
1746                account_mode: GatewayAccountMode::Test,
1747                sale_calls: AtomicUsize::new(0),
1748                outcome: Mutex::new(Some(
1749                    approved_outcome("host_txn_approved").with_diagnostics(vec![
1750                        GatewayPaymentDiagnostic::ProcessorReportedDuplicate,
1751                    ]),
1752                )),
1753            });
1754            let resolver = Arc::new(StaticResolver {
1755                gateway: resolved_gateway(account, gateway.clone()),
1756                calls: AtomicUsize::new(0),
1757            });
1758            let admission = Arc::new(PermitAdmission {
1759                calls: AtomicUsize::new(0),
1760            });
1761            let events = Arc::new(Mutex::new(Vec::new()));
1762            let coordinator = Arc::new(TestCoordinator {
1763                pool: database.pool.clone(),
1764                events: Arc::clone(&events),
1765            });
1766            let targets = Arc::new(TestTargets);
1767            let service = SubscriptionBillingService::new(
1768                database.pool.clone(),
1769                Arc::new(UnusedOffers),
1770                resolver.clone(),
1771                admission.clone(),
1772                coordinator.clone(),
1773            )
1774            .with_required_gateway_account_mode(GatewayAccountMode::Test)
1775            .with_host_charge_targets(targets);
1776            let command = ChargeHostTarget::new(
1777                syrup_rail::BillingScopeId::new(account.billing_scope_id),
1778                syrup_rail::SubscriberId::new(subscriber_id),
1779                HostChargeTargetId::new(target_id),
1780                GatewayConfigurationId::new(account.gateway_configuration_id),
1781                PaymentToken::new("tok_host_once")?,
1782                IdempotencyKey::new("host-idempotency")?,
1783                Some(BillingContact::new(
1784                    None,
1785                    None,
1786                    Some("host@example.test".into()),
1787                )?),
1788            );
1789
1790            let first = service.charge_host_target(command.clone()).await?;
1791            assert_eq!(first.status(), PaymentAttemptStatus::Approved);
1792            assert_eq!(
1793                first.gateway_diagnostics(),
1794                &[GatewayPaymentDiagnostic::ProcessorReportedDuplicate]
1795            );
1796            assert_eq!(gateway.sale_calls.load(Ordering::SeqCst), 1);
1797            assert_eq!(resolver.calls.load(Ordering::SeqCst), 1);
1798            assert_eq!(admission.calls.load(Ordering::SeqCst), 1);
1799            sqlx::query("UPDATE host_charge_targets SET status = 'reversed' WHERE id = $1")
1800                .bind(target_id)
1801                .execute(&database.pool)
1802                .await?;
1803            let approved_replay = approved_outcome("host_txn_approved")
1804                .with_diagnostics(vec![GatewayPaymentDiagnostic::ProcessorReportedDuplicate]);
1805            let reconciled_replay = apply_reconciled_host_charge_gateway_outcome(
1806                &database.pool,
1807                coordinator.as_ref(),
1808                &RefusingTransitionTargets,
1809                syrup_rail::BillingScopeId::new(account.billing_scope_id),
1810                first.attempt().identity().attempt_id(),
1811                &approved_replay,
1812            )
1813            .await?;
1814            assert_eq!(reconciled_replay.attempt(), first.attempt());
1815            assert_eq!(
1816                reconciled_replay.gateway_diagnostics(),
1817                &[GatewayPaymentDiagnostic::ProcessorReportedDuplicate]
1818            );
1819            let live_service = service
1820                .clone()
1821                .with_required_gateway_account_mode(GatewayAccountMode::Live);
1822            let replay = live_service.charge_host_target(command).await?;
1823            assert_eq!(replay.attempt(), first.attempt());
1824            assert!(replay.gateway_diagnostics().is_empty());
1825            assert_eq!(gateway.sale_calls.load(Ordering::SeqCst), 1);
1826            assert_eq!(resolver.calls.load(Ordering::SeqCst), 1);
1827            assert_eq!(admission.calls.load(Ordering::SeqCst), 1);
1828
1829            let target_status: String =
1830                sqlx::query_scalar("SELECT status FROM host_charge_targets WHERE id = $1")
1831                    .bind(target_id)
1832                    .fetch_one(&database.pool)
1833                    .await?;
1834            assert_eq!(target_status, "reversed");
1835            let charge_state: String = sqlx::query_scalar(
1836                "SELECT progression_state FROM billing_processor_charges WHERE attempt_id = $1",
1837            )
1838            .bind(first.attempt().identity().attempt_id().as_uuid())
1839            .fetch_one(&database.pool)
1840            .await?;
1841            assert_eq!(charge_state, "applied");
1842            let event_keys = events
1843                .lock()
1844                .await
1845                .iter()
1846                .map(BillingEvent::semantic_key)
1847                .collect::<Vec<_>>();
1848            assert_eq!(
1849                event_keys,
1850                vec![BillingEventKey::HostChargePaid(HostChargeTargetId::new(
1851                    target_id
1852                ))]
1853            );
1854            Ok::<_, Box<dyn Error>>(())
1855        }
1856        .await;
1857        let cleanup = database.cleanup().await;
1858        result?;
1859        cleanup?;
1860        Ok(())
1861    }
1862
1863    #[tokio::test]
1864    async fn submission_admission_validates_identity_and_skips_nonprepared_replay()
1865    -> Result<(), Box<dyn Error>> {
1866        let database = TestDatabase::start("rail_host_ident").await?;
1867        let result = async {
1868            sqlx::query(
1869                r#"
1870                CREATE TABLE host_charge_targets (
1871                    id uuid PRIMARY KEY,
1872                    billing_scope_id uuid NOT NULL,
1873                    subscriber_id uuid NOT NULL,
1874                    status text NOT NULL,
1875                    amount_cents integer NOT NULL,
1876                    currency text NOT NULL,
1877                    paid_at timestamptz
1878                )
1879                "#,
1880            )
1881            .execute(&database.pool)
1882            .await?;
1883            let account = create_gateway_account(&database.pool, "nmi").await?;
1884            let subscriber_id = Uuid::now_v7();
1885            let target_id = Uuid::now_v7();
1886            sqlx::query(
1887                "INSERT INTO host_charge_targets VALUES ($1, $2, $3, 'pending', 1250, 'USD', NULL)",
1888            )
1889            .bind(target_id)
1890            .bind(account.billing_scope_id)
1891            .bind(subscriber_id)
1892            .execute(&database.pool)
1893            .await?;
1894            let gateway = resolved_gateway(
1895                account,
1896                Arc::new(ScriptedGateway {
1897                    account_mode: GatewayAccountMode::Live,
1898                    sale_calls: AtomicUsize::new(0),
1899                    outcome: Mutex::new(None),
1900                }),
1901            );
1902            let snapshot = syrup_rail::HostChargeTargetSnapshot::new(
1903                HostChargeTargetId::new(target_id),
1904                ChargeAmount::new(1250, CurrencyCode::new("USD")?)?,
1905            );
1906            let command = ChargeHostTarget::new(
1907                syrup_rail::BillingScopeId::new(account.billing_scope_id),
1908                syrup_rail::SubscriberId::new(subscriber_id),
1909                HostChargeTargetId::new(target_id),
1910                GatewayConfigurationId::new(account.gateway_configuration_id),
1911                PaymentToken::new("tok_host_admission_identity")?,
1912                IdempotencyKey::new("host-admission-identity")?,
1913                None,
1914            );
1915            let attempt_id = PaymentAttemptId::new(Uuid::now_v7());
1916            let reservation = HostChargeReservation::from_command(
1917                &command,
1918                snapshot,
1919                &gateway,
1920                attempt_id,
1921                GatewayAccountMode::Live,
1922            )?;
1923            let mut transaction = database.pool.begin().await?;
1924            assert!(matches!(
1925                reserve_host_charge_in_transaction(&mut transaction, &TestTargets, &reservation)
1926                    .await?,
1927                HostChargeReservationOutcome::Reserved(_)
1928            ));
1929            transaction.commit().await?;
1930
1931            let wrong_mode = HostChargeReservation::from_command(
1932                &command,
1933                snapshot,
1934                &gateway,
1935                attempt_id,
1936                GatewayAccountMode::Test,
1937            )?;
1938            assert!(matches!(
1939                admit_host_charge_submission(&database.pool, &TestTargets, &wrong_mode).await,
1940                Err(HostChargeApplicationError::Store(
1941                    HostChargeStoreError::InvalidState
1942                ))
1943            ));
1944
1945            let mut transaction = database.pool.begin().await?;
1946            let attempt = crate::find_payment_attempt_by_id_in_transaction(
1947                &mut transaction,
1948                command.billing_scope_id(),
1949                attempt_id,
1950            )
1951            .await?
1952            .expect("reserved attempt");
1953            transaction.commit().await?;
1954            assert_eq!(attempt.status(), PaymentAttemptStatus::Pending);
1955            assert!(attempt.state().timestamps().submitted_at().is_none());
1956
1957            sqlx::query(
1958                "UPDATE billing_payment_attempts SET status = 'review_required' WHERE id = $1",
1959            )
1960            .bind(attempt_id.as_uuid())
1961            .execute(&database.pool)
1962            .await?;
1963            let HostChargeAdmissionOutcome::AlreadyAdmitted(review) =
1964                admit_host_charge_submission(&database.pool, &AdmissionMustNotRun, &wrong_mode)
1965                    .await?
1966            else {
1967                panic!("wrong-mode review replay must return the canonical attempt");
1968            };
1969            assert_eq!(review.status(), PaymentAttemptStatus::ReviewRequired);
1970            Ok::<_, Box<dyn Error>>(())
1971        }
1972        .await;
1973        let cleanup = database.cleanup().await;
1974        result?;
1975        cleanup?;
1976        Ok(())
1977    }
1978
1979    #[tokio::test]
1980    async fn reservation_race_replays_equivalent_contact_and_rejects_changed_contact()
1981    -> Result<(), Box<dyn Error>> {
1982        let database = TestDatabase::start("rail_host_race").await?;
1983        let result = async {
1984            sqlx::query(
1985                r#"
1986                CREATE TABLE host_charge_targets (
1987                    id uuid PRIMARY KEY,
1988                    billing_scope_id uuid NOT NULL,
1989                    subscriber_id uuid NOT NULL,
1990                    status text NOT NULL,
1991                    amount_cents integer NOT NULL,
1992                    currency text NOT NULL,
1993                    paid_at timestamptz
1994                )
1995                "#,
1996            )
1997            .execute(&database.pool)
1998            .await?;
1999            let account = create_gateway_account(&database.pool, "nmi").await?;
2000            let subscriber_id = Uuid::now_v7();
2001            let target_id = Uuid::now_v7();
2002            sqlx::query(
2003                "INSERT INTO host_charge_targets VALUES ($1, $2, $3, 'pending', 1250, 'USD', NULL)",
2004            )
2005            .bind(target_id)
2006            .bind(account.billing_scope_id)
2007            .bind(subscriber_id)
2008            .execute(&database.pool)
2009            .await?;
2010            let gateway = resolved_gateway(
2011                account,
2012                Arc::new(ScriptedGateway {
2013                    account_mode: GatewayAccountMode::Live,
2014                    sale_calls: AtomicUsize::new(0),
2015                    outcome: Mutex::new(None),
2016                }),
2017            );
2018            let snapshot = syrup_rail::HostChargeTargetSnapshot::new(
2019                HostChargeTargetId::new(target_id),
2020                ChargeAmount::new(1250, CurrencyCode::new("USD")?)?,
2021            );
2022            let command = ChargeHostTarget::new(
2023                syrup_rail::BillingScopeId::new(account.billing_scope_id),
2024                syrup_rail::SubscriberId::new(subscriber_id),
2025                HostChargeTargetId::new(target_id),
2026                GatewayConfigurationId::new(account.gateway_configuration_id),
2027                PaymentToken::new("tok_host_winner")?,
2028                IdempotencyKey::new("host-reservation-race")?,
2029                Some(BillingContact::new(
2030                    Some("Mary Ann".into()),
2031                    Some("Smith".into()),
2032                    Some("winner@example.test".into()),
2033                )?),
2034            );
2035            let winner_id = PaymentAttemptId::new(Uuid::now_v7());
2036            let winner = HostChargeReservation::from_command(
2037                &command,
2038                snapshot,
2039                &gateway,
2040                winner_id,
2041                GatewayAccountMode::Live,
2042            )?;
2043            let mut transaction = database.pool.begin().await?;
2044            let outcome =
2045                reserve_host_charge_in_transaction(&mut transaction, &TestTargets, &winner).await?;
2046            assert!(matches!(outcome, HostChargeReservationOutcome::Reserved(_)));
2047            transaction.commit().await?;
2048
2049            let retry = ChargeHostTarget::new(
2050                command.billing_scope_id(),
2051                command.subscriber_id(),
2052                command.target_id(),
2053                command.gateway_configuration_id(),
2054                PaymentToken::new("tok_host_retry")?,
2055                command.idempotency_key().clone(),
2056                command.billing_contact().cloned(),
2057            );
2058            let contender = HostChargeReservation::from_command(
2059                &retry,
2060                snapshot,
2061                &gateway,
2062                PaymentAttemptId::new(Uuid::now_v7()),
2063                GatewayAccountMode::Live,
2064            )?;
2065            let mut transaction = database.pool.begin().await?;
2066            let outcome =
2067                reserve_host_charge_in_transaction(&mut transaction, &TestTargets, &contender)
2068                    .await?;
2069            transaction.commit().await?;
2070            let HostChargeReservationOutcome::Replay(attempt) = outcome else {
2071                panic!("matching contender should replay the durable winner");
2072            };
2073            assert_eq!(attempt.identity().attempt_id(), winner_id);
2074            assert_eq!(
2075                attempt.request().billing_contact().email(),
2076                Some("winner@example.test")
2077            );
2078
2079            let changed_mode_contender = HostChargeReservation::from_command(
2080                &retry,
2081                snapshot,
2082                &gateway,
2083                PaymentAttemptId::new(Uuid::now_v7()),
2084                GatewayAccountMode::Test,
2085            )?;
2086            let mut transaction = database.pool.begin().await?;
2087            let outcome = reserve_host_charge_in_transaction(
2088                &mut transaction,
2089                &TestTargets,
2090                &changed_mode_contender,
2091            )
2092            .await?;
2093            transaction.rollback().await?;
2094            assert_eq!(
2095                outcome,
2096                HostChargeReservationOutcome::GatewayAccountModeChanged
2097            );
2098
2099            sqlx::query(
2100                "UPDATE billing_payment_attempts SET status = 'review_required' WHERE id = $1",
2101            )
2102            .bind(winner_id.as_uuid())
2103            .execute(&database.pool)
2104            .await?;
2105            let mut transaction = database.pool.begin().await?;
2106            let outcome = reserve_host_charge_in_transaction(
2107                &mut transaction,
2108                &TestTargets,
2109                &changed_mode_contender,
2110            )
2111            .await?;
2112            transaction.commit().await?;
2113            let HostChargeReservationOutcome::Replay(review) = outcome else {
2114                panic!("an unsubmitted review attempt cannot resume provider submission");
2115            };
2116            assert_eq!(review.identity().attempt_id(), winner_id);
2117            assert_eq!(review.status(), PaymentAttemptStatus::ReviewRequired);
2118
2119            let changed_contact_retry = ChargeHostTarget::new(
2120                command.billing_scope_id(),
2121                command.subscriber_id(),
2122                command.target_id(),
2123                command.gateway_configuration_id(),
2124                PaymentToken::new("tok_host_changed_contact")?,
2125                command.idempotency_key().clone(),
2126                Some(BillingContact::new(
2127                    Some("Mary".into()),
2128                    Some("Ann Smith".into()),
2129                    Some("winner@example.test".into()),
2130                )?),
2131            );
2132            let changed_contact_contender = HostChargeReservation::from_command(
2133                &changed_contact_retry,
2134                snapshot,
2135                &gateway,
2136                PaymentAttemptId::new(Uuid::now_v7()),
2137                GatewayAccountMode::Live,
2138            )?;
2139            let mut transaction = database.pool.begin().await?;
2140            let outcome = reserve_host_charge_in_transaction(
2141                &mut transaction,
2142                &TestTargets,
2143                &changed_contact_contender,
2144            )
2145            .await?;
2146            transaction.rollback().await?;
2147            assert_eq!(outcome, HostChargeReservationOutcome::IdempotencyConflict);
2148            Ok::<_, Box<dyn Error>>(())
2149        }
2150        .await;
2151        let cleanup = database.cleanup().await;
2152        result?;
2153        cleanup?;
2154        Ok(())
2155    }
2156
2157    #[tokio::test]
2158    async fn unreserved_rate_limit_avoids_attempt_and_extends_provider_cooldown()
2159    -> Result<(), Box<dyn Error>> {
2160        let database = TestDatabase::start("rail_host_rate").await?;
2161        let result = async {
2162            sqlx::query(
2163                r#"
2164                CREATE TABLE host_charge_targets (
2165                    id uuid PRIMARY KEY,
2166                    billing_scope_id uuid NOT NULL,
2167                    subscriber_id uuid NOT NULL,
2168                    status text NOT NULL,
2169                    amount_cents integer NOT NULL,
2170                    currency text NOT NULL,
2171                    paid_at timestamptz
2172                )
2173                "#,
2174            )
2175            .execute(&database.pool)
2176            .await?;
2177            let account = create_gateway_account(&database.pool, "nmi").await?;
2178            let subscriber_id = Uuid::now_v7();
2179            let target_id = Uuid::now_v7();
2180            sqlx::query(
2181                "INSERT INTO host_charge_targets VALUES ($1, $2, $3, 'pending', 1250, 'USD', NULL)",
2182            )
2183            .bind(target_id)
2184            .bind(account.billing_scope_id)
2185            .bind(subscriber_id)
2186            .execute(&database.pool)
2187            .await?;
2188
2189            let gateway = Arc::new(RateLimitedAfterReservationGateway {
2190                readiness_calls: AtomicUsize::new(0),
2191                sale_calls: AtomicUsize::new(0),
2192            });
2193            let resolver = Arc::new(StaticResolver {
2194                gateway: resolved_gateway(account, gateway.clone()),
2195                calls: AtomicUsize::new(0),
2196            });
2197            let admission = Arc::new(PermitAdmission {
2198                calls: AtomicUsize::new(0),
2199            });
2200            let service = SubscriptionBillingService::new(
2201                database.pool.clone(),
2202                Arc::new(UnusedOffers),
2203                resolver,
2204                admission.clone(),
2205                Arc::new(TestCoordinator {
2206                    pool: database.pool.clone(),
2207                    events: Arc::new(Mutex::new(Vec::new())),
2208                }),
2209            )
2210            .with_host_charge_targets(Arc::new(TestTargets));
2211            let command = ChargeHostTarget::new(
2212                syrup_rail::BillingScopeId::new(account.billing_scope_id),
2213                syrup_rail::SubscriberId::new(subscriber_id),
2214                HostChargeTargetId::new(target_id),
2215                GatewayConfigurationId::new(account.gateway_configuration_id),
2216                PaymentToken::new("tok_host_throttled")?,
2217                IdempotencyKey::new("host-throttled")?,
2218                None,
2219            );
2220
2221            let error = service
2222                .charge_host_target(command)
2223                .await
2224                .expect_err("provider throttle must be reported");
2225            assert!(matches!(
2226                error,
2227                crate::SubscriptionBillingServiceError::GatewayMutationCooldown {
2228                    scope: crate::GatewayMutationCooldownScope::Provider
2229                }
2230            ));
2231            assert_eq!(gateway.readiness_calls.load(Ordering::SeqCst), 1);
2232            assert_eq!(gateway.sale_calls.load(Ordering::SeqCst), 0);
2233
2234            let attempt_count: i64 = sqlx::query_scalar(
2235                "SELECT COUNT(*) FROM billing_payment_attempts \
2236                 WHERE billing_scope_id = $1 AND subscriber_id = $2",
2237            )
2238            .bind(account.billing_scope_id)
2239            .bind(subscriber_id)
2240            .fetch_one(&database.pool)
2241            .await?;
2242            assert_eq!(attempt_count, 0);
2243            assert_eq!(admission.calls.load(Ordering::SeqCst), 0);
2244            let cooldown_is_active: bool = sqlx::query_scalar(
2245                "SELECT rate_limited_until > clock_timestamp() FROM billing_gateway_provider_rate_limits WHERE provider_key = 'nmi'",
2246            )
2247            .fetch_one(&database.pool)
2248            .await?;
2249            assert!(cooldown_is_active);
2250            let target_status: String =
2251                sqlx::query_scalar("SELECT status FROM host_charge_targets WHERE id = $1")
2252                    .bind(target_id)
2253                    .fetch_one(&database.pool)
2254                    .await?;
2255            assert_eq!(target_status, "pending");
2256            Ok::<_, Box<dyn Error>>(())
2257        }
2258        .await;
2259        let cleanup = database.cleanup().await;
2260        result?;
2261        cleanup?;
2262        Ok(())
2263    }
2264
2265    #[tokio::test]
2266    async fn terminal_approval_race_does_not_mark_host_target_paid() -> Result<(), Box<dyn Error>> {
2267        let database = TestDatabase::start("rail_host_race").await?;
2268        let result = async {
2269            sqlx::query(
2270                r#"
2271                CREATE TABLE host_charge_targets (
2272                    id uuid PRIMARY KEY,
2273                    billing_scope_id uuid NOT NULL,
2274                    subscriber_id uuid NOT NULL,
2275                    status text NOT NULL,
2276                    amount_cents integer NOT NULL,
2277                    currency text NOT NULL,
2278                    paid_at timestamptz
2279                )
2280                "#,
2281            )
2282            .execute(&database.pool)
2283            .await?;
2284            let account = create_gateway_account(&database.pool, "nmi").await?;
2285            let subscriber_id = Uuid::now_v7();
2286            let target_id = Uuid::now_v7();
2287            sqlx::query(
2288                "INSERT INTO host_charge_targets VALUES ($1, $2, $3, 'pending', 1250, 'USD', NULL)",
2289            )
2290            .bind(target_id)
2291            .bind(account.billing_scope_id)
2292            .bind(subscriber_id)
2293            .execute(&database.pool)
2294            .await?;
2295
2296            let gateway = Arc::new(TerminalRaceGateway {
2297                pool: database.pool.clone(),
2298                sale_calls: AtomicUsize::new(0),
2299            });
2300            let resolver = Arc::new(StaticResolver {
2301                gateway: resolved_gateway(account, gateway.clone()),
2302                calls: AtomicUsize::new(0),
2303            });
2304            let events = Arc::new(Mutex::new(Vec::new()));
2305            let service = SubscriptionBillingService::new(
2306                database.pool.clone(),
2307                Arc::new(UnusedOffers),
2308                resolver,
2309                Arc::new(PermitAdmission {
2310                    calls: AtomicUsize::new(0),
2311                }),
2312                Arc::new(TestCoordinator {
2313                    pool: database.pool.clone(),
2314                    events: Arc::clone(&events),
2315                }),
2316            )
2317            .with_host_charge_targets(Arc::new(TestTargets));
2318            let payment = service
2319                .charge_host_target(ChargeHostTarget::new(
2320                    syrup_rail::BillingScopeId::new(account.billing_scope_id),
2321                    syrup_rail::SubscriberId::new(subscriber_id),
2322                    HostChargeTargetId::new(target_id),
2323                    GatewayConfigurationId::new(account.gateway_configuration_id),
2324                    PaymentToken::new("tok_host_race")?,
2325                    IdempotencyKey::new("host-race")?,
2326                    None,
2327                ))
2328                .await?;
2329
2330            assert_eq!(payment.status(), PaymentAttemptStatus::Unknown);
2331            assert_eq!(payment.attempt().status(), PaymentAttemptStatus::Failed);
2332            assert_eq!(gateway.sale_calls.load(Ordering::SeqCst), 1);
2333            assert!(events.lock().await.is_empty());
2334            let target_status: String =
2335                sqlx::query_scalar("SELECT status FROM host_charge_targets WHERE id = $1")
2336                    .bind(target_id)
2337                    .fetch_one(&database.pool)
2338                    .await?;
2339            assert_eq!(target_status, "pending");
2340            let charge_state: String = sqlx::query_scalar(
2341                "SELECT progression_state FROM billing_processor_charges WHERE attempt_id = $1",
2342            )
2343            .bind(payment.attempt().identity().attempt_id().as_uuid())
2344            .fetch_one(&database.pool)
2345            .await?;
2346            assert_eq!(charge_state, "pending");
2347            Ok::<_, Box<dyn Error>>(())
2348        }
2349        .await;
2350        let cleanup = database.cleanup().await;
2351        result?;
2352        cleanup?;
2353        Ok(())
2354    }
2355
2356    #[tokio::test]
2357    async fn matching_retry_resumes_prepared_attempt_without_readiness_loser_overwrite()
2358    -> Result<(), Box<dyn Error>> {
2359        let database = TestDatabase::start("rail_host_resume").await?;
2360        let result = async {
2361            sqlx::query(
2362                r#"
2363                CREATE TABLE host_charge_targets (
2364                    id uuid PRIMARY KEY,
2365                    billing_scope_id uuid NOT NULL,
2366                    subscriber_id uuid NOT NULL,
2367                    status text NOT NULL,
2368                    amount_cents integer NOT NULL,
2369                    currency text NOT NULL,
2370                    paid_at timestamptz
2371                )
2372                "#,
2373            )
2374            .execute(&database.pool)
2375            .await?;
2376            let account = create_gateway_account(&database.pool, "nmi").await?;
2377            let subscriber_id = Uuid::now_v7();
2378            let target_id = Uuid::now_v7();
2379            sqlx::query(
2380                "INSERT INTO host_charge_targets VALUES ($1, $2, $3, 'pending', 1250, 'USD', NULL)",
2381            )
2382            .bind(target_id)
2383            .bind(account.billing_scope_id)
2384            .bind(subscriber_id)
2385            .execute(&database.pool)
2386            .await?;
2387
2388            let (readiness_started_tx, readiness_started_rx) = oneshot::channel();
2389            let (sale_started_tx, sale_started_rx) = oneshot::channel();
2390            let gateway = Arc::new(RacingPreparedRetryGateway {
2391                readiness_calls: AtomicUsize::new(0),
2392                sale_calls: AtomicUsize::new(0),
2393                blocked_readiness_started: Mutex::new(Some(readiness_started_tx)),
2394                sale_started: Mutex::new(Some(sale_started_tx)),
2395                release_readiness: Notify::new(),
2396                release_sale: Notify::new(),
2397            });
2398            let command = ChargeHostTarget::new(
2399                syrup_rail::BillingScopeId::new(account.billing_scope_id),
2400                syrup_rail::SubscriberId::new(subscriber_id),
2401                HostChargeTargetId::new(target_id),
2402                GatewayConfigurationId::new(account.gateway_configuration_id),
2403                PaymentToken::new("tok_host_resume")?,
2404                IdempotencyKey::new("host-resume")?,
2405                None,
2406            );
2407            let resolved = resolved_gateway(account, gateway.clone());
2408            let reservation = HostChargeReservation::from_command(
2409                &command,
2410                syrup_rail::HostChargeTargetSnapshot::new(
2411                    HostChargeTargetId::new(target_id),
2412                    ChargeAmount::new(1250, CurrencyCode::new("USD")?)?,
2413                ),
2414                &resolved,
2415                PaymentAttemptId::new(Uuid::now_v7()),
2416                GatewayAccountMode::Live,
2417            )?;
2418            let mut transaction = database.pool.begin().await?;
2419            assert!(matches!(
2420                reserve_host_charge_in_transaction(&mut transaction, &TestTargets, &reservation,)
2421                    .await?,
2422                HostChargeReservationOutcome::Reserved(_)
2423            ));
2424            transaction.commit().await?;
2425            let service = SubscriptionBillingService::new(
2426                database.pool.clone(),
2427                Arc::new(UnusedOffers),
2428                Arc::new(StaticResolver {
2429                    gateway: resolved,
2430                    calls: AtomicUsize::new(0),
2431                }),
2432                Arc::new(PermitAdmission {
2433                    calls: AtomicUsize::new(0),
2434                }),
2435                Arc::new(TestCoordinator {
2436                    pool: database.pool.clone(),
2437                    events: Arc::new(Mutex::new(Vec::new())),
2438                }),
2439            )
2440            .with_host_charge_targets(Arc::new(TestTargets));
2441
2442            let first_service = service.clone();
2443            let first_command = command.clone();
2444            let first =
2445                tokio::spawn(async move { first_service.charge_host_target(first_command).await });
2446            readiness_started_rx.await?;
2447            let retry = ChargeHostTarget::new(
2448                command.billing_scope_id(),
2449                command.subscriber_id(),
2450                command.target_id(),
2451                command.gateway_configuration_id(),
2452                PaymentToken::new("tok_host_resume_retry")?,
2453                command.idempotency_key().clone(),
2454                command.billing_contact().cloned(),
2455            );
2456            let second_service = service.clone();
2457            let second =
2458                tokio::spawn(async move { second_service.charge_host_target(retry).await });
2459            sale_started_rx.await?;
2460            gateway.release_readiness.notify_one();
2461            let first = first.await??;
2462            assert_eq!(first.status(), PaymentAttemptStatus::Pending);
2463            assert!(
2464                first
2465                    .attempt()
2466                    .state()
2467                    .timestamps()
2468                    .submitted_at()
2469                    .is_some()
2470            );
2471            gateway.release_sale.notify_one();
2472            let second = second.await??;
2473            assert_eq!(second.status(), PaymentAttemptStatus::Approved);
2474            assert_eq!(first.attempt().identity(), second.attempt().identity());
2475            assert_eq!(gateway.sale_calls.load(Ordering::SeqCst), 1);
2476            Ok::<_, Box<dyn Error>>(())
2477        }
2478        .await;
2479        let cleanup = database.cleanup().await;
2480        result?;
2481        cleanup?;
2482        Ok(())
2483    }
2484}