use std::future::Future;
use std::sync::Arc;
use std::time::Duration;
use datafusion::error::DataFusionError;
use polyc_projection_artifact::{ArtifactReadError, ManifestOpenError, ReadContractError};
use polyc_state::error::{RetryClass, StateError};
use polyc_state::immutable::ObjectError;
use polyc_state::projection::ProjectionCatalogError;
use polyc_state::projection::artifact::ArtifactAdmissionError;
use polyc_state::query_audit::{
ErrorClass, QueryAuditError, QueryCompletion, QueryOutcome, SourceSnapshot,
};
#[cfg(test)]
use tokio::sync::Notify;
use tokio::sync::oneshot;
use tokio_util::sync::CancellationToken;
use super::CoreExecutionError;
use super::latch::TerminalLatch;
use crate::core_resolution::{
CoreCompletionContext, CoreExecutionPermit, CoreMetadataAuthority, CoreResolutionError,
};
const COMPLETION_TIMEOUT: Duration = Duration::from_secs(5);
const ATTEMPT_TIMEOUT: Duration = Duration::from_millis(500);
const MAX_ATTEMPTS: usize = 3;
const RETRY_BACKOFF: Duration = Duration::from_millis(10);
const SETTLEMENT_RESERVE: Duration = Duration::from_millis(1500);
struct FinishRequest {
outcome: QueryOutcome,
duration: Duration,
response: oneshot::Sender<Result<(), CoreExecutionError>>,
}
pub(crate) struct PermitGuardian {
request: Option<oneshot::Sender<FinishRequest>>,
cancellation: CancellationToken,
latch: Arc<TerminalLatch>,
source: polyc_state::query_audit::SourceSnapshot,
started: tokio::time::Instant,
#[cfg(test)]
dispatch_gate: Arc<GuardianTestGate>,
armed: bool,
}
impl PermitGuardian {
pub(crate) fn new(
permit: CoreExecutionPermit,
metadata: Arc<dyn CoreMetadataAuthority>,
) -> Self {
let (request, receiver) = oneshot::channel();
let cancellation = CancellationToken::new();
let latch = Arc::new(TerminalLatch::default());
let source = permit.source().clone();
let started = tokio::time::Instant::now();
#[cfg(test)]
let dispatch_gate = Arc::new(GuardianTestGate::default());
tokio::spawn(guard_permit(
permit,
metadata,
cancellation.clone(),
Arc::clone(&latch),
started,
#[cfg(test)]
Arc::clone(&dispatch_gate),
receiver,
));
Self {
request: Some(request),
cancellation,
latch,
source,
started,
#[cfg(test)]
dispatch_gate,
armed: true,
}
}
pub(crate) fn cancellation(&self) -> CancellationToken {
self.cancellation.clone()
}
pub(crate) fn latch(&self) -> Arc<TerminalLatch> {
Arc::clone(&self.latch)
}
pub(crate) const fn source(&self) -> &polyc_state::query_audit::SourceSnapshot {
&self.source
}
pub(crate) async fn finish(mut self, outcome: QueryOutcome) -> Result<(), CoreExecutionError> {
let (response, completed) = oneshot::channel();
let request = FinishRequest {
outcome,
duration: self.started.elapsed(),
response,
};
self.armed = false;
self.request
.take()
.ok_or(CoreExecutionError::AuditCompletionUnavailable)?
.send(request)
.map_err(|_| CoreExecutionError::AuditCompletionUnavailable)?;
completed
.await
.unwrap_or(Err(CoreExecutionError::AuditCompletionUnavailable))
}
pub(crate) async fn fail(self, class: ErrorClass) -> Result<(), CoreExecutionError> {
self.finish(QueryOutcome::Failed(class)).await
}
pub(crate) fn abandon(mut self) {
self.armed = false;
drop(self.request.take());
}
#[cfg(test)]
pub(crate) fn pause_dispatch(&self) -> GuardianDispatchPause {
self.dispatch_gate
.paused
.store(true, std::sync::atomic::Ordering::SeqCst);
GuardianDispatchPause {
gate: Arc::clone(&self.dispatch_gate),
}
}
}
#[cfg(test)]
#[derive(Default)]
struct GuardianTestGate {
paused: std::sync::atomic::AtomicBool,
waiting: Notify,
resume: Notify,
}
#[cfg(test)]
pub(crate) struct GuardianDispatchPause {
gate: Arc<GuardianTestGate>,
}
#[cfg(test)]
impl GuardianDispatchPause {
pub(crate) async fn wait(&self) {
self.gate.waiting.notified().await;
}
pub(crate) fn resume(&self) {
self.gate
.paused
.store(false, std::sync::atomic::Ordering::SeqCst);
self.gate.resume.notify_waiters();
}
}
impl std::fmt::Debug for PermitGuardian {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("PermitGuardian")
.field("armed", &self.armed)
.finish_non_exhaustive()
}
}
impl Drop for PermitGuardian {
fn drop(&mut self) {
if self.armed {
self.latch.cancel();
self.cancellation.cancel();
}
}
}
async fn guard_permit(
permit: CoreExecutionPermit,
metadata: Arc<dyn CoreMetadataAuthority>,
cancellation: CancellationToken,
latch: Arc<TerminalLatch>,
started: tokio::time::Instant,
#[cfg(test)] dispatch_gate: Arc<GuardianTestGate>,
mut receiver: oneshot::Receiver<FinishRequest>,
) {
let source = permit.source().clone();
let operation = CoreCompletionContext::server_owned(COMPLETION_TIMEOUT);
let mut response = None;
let mut report_possible = true;
let abandoned = tokio::select! {
biased;
request = &mut receiver => {
report_possible = false;
match request {
Ok(request) => {
let _ = latch.report(request.outcome, request.duration, &source);
response = Some(request.response);
false
}
Err(_) => !latch.is_settled(),
}
}
() = cancellation.cancelled() => {
latch.cancel();
false
}
};
if abandoned {
return;
}
if response.is_none() && report_possible {
response = await_late_report(&latch, &operation, &mut receiver, &source).await;
}
dispatch_boundary(
#[cfg(test)]
dispatch_gate.as_ref(),
)
.await;
if cancellation.is_cancelled() {
latch.cancel();
}
let completion = latch.dispatch(&source, started.elapsed());
let result = settle_completion(&operation, metadata, permit, completion).await;
if let Some(response) = response {
let _ = response.send(result);
}
}
async fn await_late_report(
latch: &TerminalLatch,
operation: &CoreCompletionContext,
receiver: &mut oneshot::Receiver<FinishRequest>,
source: &SourceSnapshot,
) -> Option<oneshot::Sender<Result<(), CoreExecutionError>>> {
let headroom = report_headroom(operation)?;
let Ok(Ok(request)) = tokio::time::timeout(headroom, receiver).await else {
return None;
};
let _ = latch.report(request.outcome, request.duration, source);
Some(request.response)
}
fn report_headroom(operation: &CoreCompletionContext) -> Option<Duration> {
let remaining = operation.remaining().ok()?;
let headroom = remaining.checked_sub(SETTLEMENT_RESERVE)?;
(!headroom.is_zero()).then_some(headroom)
}
#[cfg(not(test))]
fn dispatch_boundary() -> std::future::Ready<()> {
std::future::ready(())
}
#[cfg(test)]
async fn dispatch_boundary(gate: &GuardianTestGate) {
if gate.paused.load(std::sync::atomic::Ordering::SeqCst) {
gate.waiting.notify_one();
gate.resume.notified().await;
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ReceiptVerdict {
Settled,
Absent,
Crossed,
Unknown,
}
async fn settle_completion(
operation: &CoreCompletionContext,
metadata: Arc<dyn CoreMetadataAuthority>,
permit: CoreExecutionPermit,
completion: QueryCompletion,
) -> Result<(), CoreExecutionError> {
let command = permit.into_completion(completion)?;
for attempt in 0..MAX_ATTEMPTS {
operation
.check()
.map_err(|_| CoreExecutionError::AuditCompletionUnavailable)?;
let result =
bounded_terminal_call(operation, metadata.complete_audit(operation, &command)).await;
let receipt_may_exist = match result {
Some(Ok(receipt)) if receipt.answers(command.metadata()) => return Ok(()),
Some(Ok(_)) => return Err(CoreExecutionError::AuditReceiptMismatch),
Some(Err(error)) => match completion_retry_class(&error) {
RetryClass::Terminal => return Err(CoreExecutionError::AuditCompletionUnavailable),
RetryClass::Ambiguous => true,
RetryClass::Transient => false,
},
None => true,
};
if receipt_may_exist {
match confirm_recorded_receipt(operation, metadata.as_ref(), &command).await {
ReceiptVerdict::Settled => return Ok(()),
ReceiptVerdict::Crossed => return Err(CoreExecutionError::AuditReceiptMismatch),
ReceiptVerdict::Absent | ReceiptVerdict::Unknown => {}
}
}
if attempt + 1 < MAX_ATTEMPTS {
tokio::time::sleep(RETRY_BACKOFF).await;
}
}
Err(CoreExecutionError::AuditCompletionUnavailable)
}
async fn confirm_recorded_receipt(
operation: &CoreCompletionContext,
metadata: &dyn CoreMetadataAuthority,
command: &crate::core_resolution::CoreCompletionCommand,
) -> ReceiptVerdict {
match bounded_terminal_call(operation, metadata.completion_receipt(operation, command)).await {
Some(Ok(Some(receipt))) if receipt.answers(command.metadata()) => ReceiptVerdict::Settled,
Some(Ok(Some(_))) => ReceiptVerdict::Crossed,
Some(Ok(None)) => ReceiptVerdict::Absent,
Some(Err(error)) if completion_retry_class(&error) == RetryClass::Terminal => {
ReceiptVerdict::Crossed
}
Some(Err(_)) | None => ReceiptVerdict::Unknown,
}
}
async fn bounded_terminal_call<T>(
operation: &CoreCompletionContext,
future: impl Future<Output = T>,
) -> Option<T> {
let remaining = operation.remaining().ok()?;
tokio::time::timeout(remaining.min(ATTEMPT_TIMEOUT), future)
.await
.ok()
}
const fn completion_retry_class(error: &CoreResolutionError) -> RetryClass {
match error {
CoreResolutionError::Audit(error) => error.retry_class(),
CoreResolutionError::State(error) => error.retry_class(),
CoreResolutionError::Projection(error) => error.retry_class(),
CoreResolutionError::Unavailable
| CoreResolutionError::Statement(_)
| CoreResolutionError::DataFusion(_)
| CoreResolutionError::NoSourceDependency
| CoreResolutionError::UnknownDependency(_)
| CoreResolutionError::ParameterMismatch
| CoreResolutionError::InvalidComposition
| CoreResolutionError::InvalidAttribution
| CoreResolutionError::EmptyConversationIdentity
| CoreResolutionError::FreshnessUnsupported { .. }
| CoreResolutionError::InvalidBounds
| CoreResolutionError::CrossedPermit
| CoreResolutionError::CompletionReceiptMismatch
| CoreResolutionError::MissingSource(_)
| CoreResolutionError::SourceVectorMismatch
| CoreResolutionError::SourceMismatch(_)
| CoreResolutionError::MissingProjection(_)
| CoreResolutionError::Superseded(_)
| CoreResolutionError::IncompatibleDescriptor(_)
| CoreResolutionError::DuplicatePartition
| CoreResolutionError::DuplicateDescriptor => RetryClass::Terminal,
}
}
pub(crate) fn classify_error(error: &CoreExecutionError) -> ErrorClass {
match error {
CoreExecutionError::AuthorityNarrowed => ErrorClass::Denied,
CoreExecutionError::Deadline => ErrorClass::Deadline,
CoreExecutionError::Cancelled => ErrorClass::Cancelled,
CoreExecutionError::ReleaseBound { .. } | CoreExecutionError::SourceDecodeBound { .. } => {
ErrorClass::Bounds
}
CoreExecutionError::SourceChanged(_)
| CoreExecutionError::ExecutionAdmissionClosed
| CoreExecutionError::AuditCompletionUnavailable => ErrorClass::Unavailable,
CoreExecutionError::Artifact(error) => classify_artifact(error),
CoreExecutionError::Manifest(error) => classify_manifest(error),
CoreExecutionError::Contract(error) => classify_contract(error),
CoreExecutionError::Resolution(error) => classify_resolution(error),
CoreExecutionError::DataFusion(error) => classify_datafusion(error),
CoreExecutionError::LegacyProviderUnavailable => ErrorClass::Malformed,
CoreExecutionError::ParquetContract(_)
| CoreExecutionError::Profile(_)
| CoreExecutionError::PlanIdentityMismatch
| CoreExecutionError::AuditReceiptMismatch
| CoreExecutionError::InvalidComposition(_)
| CoreExecutionError::RealmMismatch
| CoreExecutionError::Parquet(_)
| CoreExecutionError::Arrow(_) => ErrorClass::Internal,
}
}
const fn classify_datafusion(error: &DataFusionError) -> ErrorClass {
match error {
DataFusionError::ResourcesExhausted(_) => ErrorClass::Bounds,
_ => ErrorClass::Internal,
}
}
pub(crate) fn classify_resolution(error: &CoreResolutionError) -> ErrorClass {
match error {
CoreResolutionError::State(error) => classify_state(error),
CoreResolutionError::Audit(error) => classify_audit(error),
CoreResolutionError::Projection(error) => classify_projection(error),
CoreResolutionError::MissingSource(_)
| CoreResolutionError::MissingProjection(_)
| CoreResolutionError::Superseded(_) => ErrorClass::Unavailable,
CoreResolutionError::InvalidBounds => ErrorClass::Bounds,
CoreResolutionError::Statement(_)
| CoreResolutionError::DataFusion(_)
| CoreResolutionError::NoSourceDependency
| CoreResolutionError::UnknownDependency(_)
| CoreResolutionError::ParameterMismatch
| CoreResolutionError::EmptyConversationIdentity
| CoreResolutionError::FreshnessUnsupported { .. } => ErrorClass::Malformed,
CoreResolutionError::Unavailable
| CoreResolutionError::InvalidComposition
| CoreResolutionError::InvalidAttribution
| CoreResolutionError::CrossedPermit
| CoreResolutionError::CompletionReceiptMismatch
| CoreResolutionError::SourceVectorMismatch
| CoreResolutionError::SourceMismatch(_)
| CoreResolutionError::IncompatibleDescriptor(_)
| CoreResolutionError::DuplicatePartition
| CoreResolutionError::DuplicateDescriptor => ErrorClass::Internal,
}
}
const fn classify_state(error: &StateError) -> ErrorClass {
match error {
StateError::Denied { .. } => ErrorClass::Denied,
StateError::DeadlineExpired { .. } => ErrorClass::Deadline,
StateError::Cancelled { .. } => ErrorClass::Cancelled,
StateError::BoundsExceeded { .. } => ErrorClass::Bounds,
StateError::Unavailable { .. }
| StateError::AmbiguousOutcome { .. }
| StateError::DuplicateCommand { .. }
| StateError::PartitionHeld { .. }
| StateError::RevisionConflict { .. }
| StateError::IncarnationConflict { .. }
| StateError::SourceIncarnationChanged { .. }
| StateError::RetiredRevision { .. }
| StateError::CompactedRange { .. }
| StateError::SnapshotUnavailable { .. }
| StateError::JournalDamaged => ErrorClass::Unavailable,
StateError::Malformed { .. } => ErrorClass::Malformed,
StateError::StaleFence { .. } | StateError::DigestConflict { .. } => ErrorClass::Internal,
}
}
const fn classify_audit(error: &QueryAuditError) -> ErrorClass {
match error {
QueryAuditError::State(error) => classify_state(error),
QueryAuditError::NoRecordedIntent { .. } => ErrorClass::Internal,
}
}
fn classify_projection(error: &ProjectionCatalogError) -> ErrorClass {
match error {
ProjectionCatalogError::State(error) => classify_state(error),
ProjectionCatalogError::Object(error) => classify_object(error),
ProjectionCatalogError::Artifact(error) => classify_admission(error),
ProjectionCatalogError::UnknownManifest { .. } => ErrorClass::Unavailable,
ProjectionCatalogError::StalePublisher { .. }
| ProjectionCatalogError::GenerationConflict { .. }
| ProjectionCatalogError::NonMonotonic { .. }
| ProjectionCatalogError::CursorRegression { .. }
| ProjectionCatalogError::ManifestConflict { .. } => ErrorClass::Internal,
}
}
const fn classify_object(error: &ObjectError) -> ErrorClass {
match error {
ObjectError::State(error) => classify_state(error),
ObjectError::UnknownGeneration { .. } => ErrorClass::Unavailable,
ObjectError::ProtectionUnavailable { .. }
| ObjectError::StaleGeneration { .. }
| ObjectError::NotMonotonic { .. }
| ObjectError::Retained { .. }
| ObjectError::CurrentGeneration { .. }
| ObjectError::DescriptorConflict { .. } => ErrorClass::Internal,
}
}
const fn classify_admission(error: &ArtifactAdmissionError) -> ErrorClass {
match error {
ArtifactAdmissionError::Malformed(error) => classify_state(error),
ArtifactAdmissionError::ProtectionUnavailable { .. }
| ArtifactAdmissionError::ContentMismatch { .. }
| ArtifactAdmissionError::Signature { .. }
| ArtifactAdmissionError::Disagreement { .. } => ErrorClass::Internal,
}
}
const fn classify_artifact(error: &ArtifactReadError) -> ErrorClass {
match error {
ArtifactReadError::Unavailable { .. } | ArtifactReadError::NotFound { .. } => {
ErrorClass::Unavailable
}
ArtifactReadError::VerificationBudgetExceeded { .. } => ErrorClass::Bounds,
ArtifactReadError::Refused { .. }
| ArtifactReadError::GenerationMismatch { .. }
| ArtifactReadError::DigestMismatch { .. }
| ArtifactReadError::MetadataLengthMismatch { .. }
| ArtifactReadError::RangeLengthMismatch { .. }
| ArtifactReadError::RangeOutOfBounds { .. } => ErrorClass::Internal,
}
}
const fn classify_manifest(error: &ManifestOpenError) -> ErrorClass {
match error {
ManifestOpenError::Read(error) => classify_artifact(error),
ManifestOpenError::TooLarge { .. } => ErrorClass::Bounds,
ManifestOpenError::NamespaceNotConfigured { .. }
| ManifestOpenError::TopologyMismatch { .. }
| ManifestOpenError::ProtectionUnavailable { .. }
| ManifestOpenError::DescriptorMismatch { .. }
| ManifestOpenError::Verification(_) => ErrorClass::Internal,
}
}
const fn classify_contract(error: &ReadContractError) -> ErrorClass {
match error {
ReadContractError::MissingTable { .. } => ErrorClass::Unavailable,
ReadContractError::InvalidVerificationBudget { .. }
| ReadContractError::EmptyRealm { .. }
| ReadContractError::SharedNamespace { .. }
| ReadContractError::NamespaceNotConfigured { .. }
| ReadContractError::ProtectionUnavailable { .. } => ErrorClass::Internal,
}
}
#[cfg(test)]
mod tests {
use super::*;
use polyc_state::id::PartitionId;
use polyc_state::projection::artifact::AccessRealm;
use polyc_state::revision::{JournalPosition, Revision};
#[test]
fn damaged_journal_storage_is_unavailable_not_malformed_or_internal() {
let class = classify_state(&StateError::JournalDamaged);
assert_eq!(class, ErrorClass::Unavailable);
assert_ne!(
class,
ErrorClass::Malformed,
"no request a caller can send repairs damaged storage"
);
assert_ne!(
class,
ErrorClass::Internal,
"the fault is the state plane's, not the query plane's"
);
assert_eq!(
StateError::JournalDamaged.retry_class(),
polyc_state::error::RetryClass::Terminal,
"an operator repairs this; waiting never clears it"
);
assert!(!StateError::JournalDamaged.is_retry_safe());
}
#[test]
fn a_corrupt_or_disagreeing_artifact_is_internal_not_malformed() {
for error in [
ArtifactReadError::DigestMismatch {
key: "content-free".to_owned(),
generation: 1,
},
ArtifactReadError::GenerationMismatch {
expected: 2,
observed: 3,
key: "content-free".to_owned(),
},
ArtifactReadError::MetadataLengthMismatch {
key: "content-free".to_owned(),
expected: 4,
observed: 5,
},
ArtifactReadError::RangeLengthMismatch {
key: "content-free".to_owned(),
offset: 0,
expected: 4,
observed: 5,
},
ArtifactReadError::RangeOutOfBounds {
key: "content-free".to_owned(),
offset: 9,
requested: 4,
byte_len: 2,
},
ArtifactReadError::Refused {
reason: "content-free".to_owned(),
},
] {
assert_eq!(
classify_artifact(&error),
ErrorClass::Internal,
"a stored-content or deployment disagreement is never Malformed"
);
}
}
#[test]
fn an_absent_source_is_unavailable_and_a_declared_ceiling_is_bounds() {
assert_eq!(
classify_artifact(&ArtifactReadError::NotFound {
key: "content-free".to_owned(),
generation: 1,
}),
ErrorClass::Unavailable
);
assert_eq!(
classify_artifact(&ArtifactReadError::Unavailable {
reason: "content-free".to_owned(),
}),
ErrorClass::Unavailable
);
assert_eq!(
classify_artifact(&ArtifactReadError::VerificationBudgetExceeded {
key: "content-free".to_owned(),
byte_len: 2,
max_byte_len: 1,
}),
ErrorClass::Bounds
);
}
#[test]
fn nested_state_classes_remain_distinct() {
let family = polyc_state::id::OperationFamily::new("query-test");
assert_eq!(
classify_state(&StateError::Denied {
family: family.clone(),
}),
ErrorClass::Denied
);
assert_eq!(
classify_state(&StateError::Cancelled { family }),
ErrorClass::Cancelled
);
}
#[test]
fn a_retired_or_compacted_source_is_unavailable_not_malformed() {
assert_eq!(
classify_state(&StateError::RetiredRevision {
requested: Revision::new(4),
earliest: Revision::new(9),
}),
ErrorClass::Unavailable
);
assert_eq!(
classify_state(&StateError::CompactedRange {
partition: PartitionId::new("conv-a"),
requested: JournalPosition::new(4),
earliest: JournalPosition::new(9),
}),
ErrorClass::Unavailable
);
}
#[test]
fn a_deployment_fault_is_internal_and_a_missing_table_is_unavailable() {
assert_eq!(
classify_contract(&ReadContractError::NamespaceNotConfigured {
realm: AccessRealm::Visible,
namespace: "absent".to_owned(),
}),
ErrorClass::Internal
);
assert_eq!(
classify_contract(&ReadContractError::MissingTable { table: "messages" }),
ErrorClass::Unavailable
);
}
#[test]
fn only_the_memory_pool_makes_an_engine_error_a_ceiling() {
assert_eq!(
classify_datafusion(&DataFusionError::ResourcesExhausted(
"content-free".to_owned()
)),
ErrorClass::Bounds
);
assert_eq!(
classify_datafusion(&DataFusionError::Internal("content-free".to_owned())),
ErrorClass::Internal
);
}
}