use super::*;
use saddle_core::{CleanupOutcome, OperationOutcome};
use saddle_observability::{DiagnosticSubmission, Observer};
pub(crate) struct ScopeState {
primary: Option<ReservedRequestFailure>,
cleanup: Option<ReservedRequestFailure>,
database: Option<ReservedRequestFailure>,
database_more: Option<Retained<OperationFailureStorage>>,
finished: bool,
stage_started: bool,
stage_finished: bool,
stop: Option<crate::profusegw::ProfuseGwScopeStop>,
panicked: bool,
next: Option<Retained<ScopeStorage>>,
}
#[repr(C)]
struct ScopeContent {
state: Mutex<ScopeState>,
view: ReservedRequestView,
}
#[repr(C)]
pub(crate) struct ScopeStorage {
content: ScopeContent,
permit: StoragePermit,
}
#[repr(C)]
struct OperationFailureContent {
failure: Mutex<Option<ReservedRequestFailure>>,
next: Option<Retained<OperationFailureStorage>>,
}
#[repr(C)]
struct OperationFailureStorage {
content: OperationFailureContent,
permit: StoragePermit,
}
pub struct ReservedScopeObservation {
pub(crate) storage: Retained<ScopeStorage>,
task: Retained<TaskStatusStorage>,
}
pub struct ReservedScopeOperation<'a> {
observation: &'a ReservedScopeObservation,
view: ReservedRequestView,
}
impl ReservedScopeOperation<'_> {
pub fn diagnostic_view(&self) -> ReservedRequestView { self.view.clone() }
#[cfg(test)]
pub(crate) fn test_original(
&self,
error: &(dyn std::error::Error + 'static),
output: Option<&EmergencyDiagnosticHandle>,
) -> ReservedRequestFailure {
self.view.source_error(
error,
output,
saddle_core::DiagnosticStage::RequestDb,
RootRequestEvent::Database,
Default::default(),
)
}
pub fn source_cleanup_existing_error(
&self,
error: &(dyn std::error::Error + 'static),
diagnostic: saddle_core::Diagnostic,
classification: saddle_core::DiagnosticCode,
output: Option<&EmergencyDiagnosticHandle>,
facts: RootOutcomeFacts,
) -> Result<saddle_core::DiagnosticOccurrence, ReservedRequestFailure> {
let failure = self.view.source_existing_error(
error,
diagnostic,
classification,
output,
RootRequestEvent::Database,
facts,
);
self.retain_cleanup_failure(failure)
}
pub fn retain_cleanup_failure(
&self,
failure: ReservedRequestFailure,
) -> Result<saddle_core::DiagnosticOccurrence, ReservedRequestFailure> {
if !self.view.0.view.same_view(&failure.source.0.view) {
return Err(failure);
}
let occurrence = failure.occurrence();
let mut state = self
.observation
.storage
.content
.state
.lock()
.unwrap_or_else(|p| p.into_inner());
if state.stage_finished || state.cleanup.is_some() {
return Err(failure);
}
state.cleanup = Some(failure);
Ok(occurrence)
}
pub fn source_error_with_facts(
&self,
error: &(dyn std::error::Error + 'static),
diagnostic: saddle_core::BoundedDiagnostic,
classification: saddle_core::DiagnosticCode,
output: Option<&EmergencyDiagnosticHandle>,
facts: RootOutcomeFacts,
) -> Result<saddle_core::DiagnosticOccurrence, ReservedRequestFailure> {
let failure = self.view.source_error_with_facts(
error,
diagnostic,
classification,
output,
RootRequestEvent::Database,
facts,
);
self.retain_failure(failure)
}
pub fn source_description<D: std::fmt::Display + std::fmt::Debug>(
&self,
description: &D,
diagnostic: saddle_core::BoundedDiagnostic,
classification: saddle_core::DiagnosticCode,
output: Option<&EmergencyDiagnosticHandle>,
facts: RootOutcomeFacts,
) -> Result<saddle_core::DiagnosticOccurrence, ReservedRequestFailure> {
let failure = self.view.source_description(
description,
diagnostic,
classification,
output,
RootRequestEvent::Database,
facts,
);
self.retain_failure(failure)
}
pub fn source_existing_error(
&self,
error: &(dyn std::error::Error + 'static),
diagnostic: saddle_core::Diagnostic,
classification: saddle_core::DiagnosticCode,
output: Option<&EmergencyDiagnosticHandle>,
facts: RootOutcomeFacts,
) -> Result<saddle_core::DiagnosticOccurrence, ReservedRequestFailure> {
let failure = self.view.source_existing_error(
error,
diagnostic,
classification,
output,
RootRequestEvent::Database,
facts,
);
self.retain_failure(failure)
}
pub fn source_existing_description<D: std::fmt::Display + std::fmt::Debug>(
&self,
description: &D,
diagnostic: saddle_core::Diagnostic,
classification: saddle_core::DiagnosticCode,
output: Option<&EmergencyDiagnosticHandle>,
facts: RootOutcomeFacts,
) -> Result<saddle_core::DiagnosticOccurrence, ReservedRequestFailure> {
let failure = self.view.source_existing_description(
description,
diagnostic,
classification,
output,
RootRequestEvent::Database,
facts,
);
self.retain_failure(failure)
}
pub fn retain_failure(
&self,
failure: ReservedRequestFailure,
) -> Result<saddle_core::DiagnosticOccurrence, ReservedRequestFailure> {
if !self.view.0.view.same_view(&failure.source.0.view) {
return Err(failure);
}
let occurrence = failure.occurrence();
let mut state = self
.observation
.storage
.content
.state
.lock()
.unwrap_or_else(|p| p.into_inner());
if state.stage_finished {
return Err(failure);
}
if state.database.is_none() {
state.database = Some(failure);
} else {
let demand = match shared_prefix::<OperationFailureContent>()
.and_then(|layout| StorageDemand::embedded(layout, Layout::new::<()>(), &[]))
{
Ok(demand) => demand,
Err(_) => return Err(failure),
};
let permit = match self.view.0.permit.try_reserve(demand) {
Ok(permit) => permit,
Err(_) => return Err(failure),
};
state.database_more = Some(Retained::new(OperationFailureStorage {
content: OperationFailureContent {
failure: Mutex::new(Some(failure)),
next: state.database_more.take(),
},
permit,
}));
}
Ok(occurrence)
}
}
#[must_use = "complete the matching observation after physical cleanup; task retains sources if abandoned"]
pub struct ReservedScopeCompletion {
pub(crate) storage: Retained<ScopeStorage>,
}
#[must_use]
pub struct ReservedSupervisionFailure {
occurrence: saddle_core::DiagnosticOccurrence,
facts: ReservedScopeSupervisionFacts,
}
impl ReservedSupervisionFailure {
pub fn occurrence(&self) -> saddle_core::DiagnosticOccurrence {
self.occurrence
}
pub fn supervision(&self) -> ReservedScopeSupervisionFacts {
self.facts
}
}
#[derive(Clone, Copy, Debug)]
pub struct ReservedScopeSupervisionFacts {
pub stop: Option<crate::profusegw::ProfuseGwScopeStop>,
pub panicked: bool,
pub cleanup_failed: bool,
}
impl ReservedScopeCompletion {
pub fn into_failure(self) -> Result<ReservedSupervisionFailure, Self> {
let state = self
.storage
.content
.state
.lock()
.unwrap_or_else(|p| p.into_inner());
let source = state
.primary
.as_ref()
.or(state.database.as_ref())
.or(state.cleanup.as_ref());
let result = if state.finished {
source.map(|source| ReservedSupervisionFailure {
occurrence: source.occurrence(),
facts: ReservedScopeSupervisionFacts {
stop: state.stop,
panicked: state.panicked,
cleanup_failed: state.cleanup.is_some(),
},
})
} else {
None
};
drop(state);
result.ok_or(self)
}
pub fn supervision(&self) -> ReservedScopeSupervisionFacts {
let state = self
.storage
.content
.state
.lock()
.unwrap_or_else(|p| p.into_inner());
ReservedScopeSupervisionFacts {
stop: state.stop,
panicked: state.panicked,
cleanup_failed: state.cleanup.is_some(),
}
}
}
pub struct ReservedSupervisedStage<'a> {
stage: ReservedActiveStage<'a>,
storage: &'a Retained<ScopeStorage>,
task: &'a Retained<TaskStatusStorage>,
}
impl ReservedTaskContext {
#[cfg(test)]
pub(crate) fn test_scope_layouts() {
let scope = StorageDemand::embedded(
shared_prefix::<ScopeContent>().unwrap(),
Layout::new::<()>(),
&[],
)
.unwrap();
let failure = StorageDemand::embedded(
shared_prefix::<OperationFailureContent>().unwrap(),
Layout::new::<()>(),
&[],
)
.unwrap();
assert_eq!(
scope.bytes(),
shared_prefix::<ScopeStorage>().unwrap().size()
);
assert_eq!(
failure.bytes(),
shared_prefix::<OperationFailureStorage>().unwrap().size()
);
}
#[cfg(test)]
pub(crate) fn test_scope_nodes(&self) -> usize {
let mut next = self
.status
.lock()
.unwrap_or_else(|p| p.into_inner())
.scopes
.clone();
let mut count = 0;
while let Some(node) = next {
count += 1;
next = node
.content
.state
.lock()
.unwrap_or_else(|p| p.into_inner())
.next
.clone();
}
count
}
#[cfg(test)]
pub(crate) fn test_storage_available(&self) -> usize {
let mut low = 0;
let mut high = 65_536;
while low < high {
let mid = (low + high + 1) / 2;
let demand = StorageDemand::embedded(
Layout::new::<()>(),
Layout::new::<()>(),
&[(Layout::from_size_align(mid, 1).unwrap(), 1)],
)
.unwrap();
match self.view.0.permit.try_reserve(demand) {
Ok(permit) => {
drop(permit);
low = mid;
}
Err(_) => high = mid - 1,
}
}
low
}
pub(crate) fn prepare_scope_observation(
&self,
view: ReservedRequestView,
) -> Result<ReservedScopeObservation, ReservedContextError> {
if !self.view.same_request(&view) {
return Err(ReservedContextError::Context(ContextConflict::ForeignRoot));
}
let demand = StorageDemand::embedded(
shared_prefix::<ScopeContent>().map_err(ReservedContextError::Storage)?,
Layout::new::<()>(),
&[],
)
.map_err(ReservedContextError::Storage)?;
let permit = view
.0
.permit
.try_reserve(demand)
.map_err(ReservedContextError::Storage)?;
let mut task = self.status.lock().unwrap_or_else(|p| p.into_inner());
let storage = Retained::new(ScopeStorage {
content: ScopeContent {
state: Mutex::new(ScopeState {
primary: None,
cleanup: None,
database: None,
database_more: None,
finished: false,
stage_started: false,
stage_finished: false,
stop: None,
panicked: false,
next: task.scopes.take(),
}),
view,
},
permit,
});
task.scopes = Some(storage.clone());
Ok(ReservedScopeObservation {
storage,
task: self.status.clone(),
})
}
}
impl ReservedScopeObservation {
pub fn operation(
&self,
operation: RegisteredContextOperation,
) -> Result<ReservedScopeOperation<'_>, ReservedContextError> {
let state = self
.storage
.content
.state
.lock()
.unwrap_or_else(|p| p.into_inner());
if state.stage_finished {
return Err(ReservedContextError::Context(ContextConflict::ForeignRoot));
}
let view = self.storage.content.view.with_db_operation(operation)?;
drop(state);
Ok(ReservedScopeOperation {
observation: self,
view,
})
}
#[cfg(test)]
pub(crate) fn exhaust_test_storage(&self) -> StoragePermit {
let demand = |n| {
StorageDemand::embedded(
Layout::new::<()>(),
Layout::new::<()>(),
&[(Layout::from_size_align(n, 1).unwrap(), 1)],
)
.unwrap()
};
let mut low = 0;
let mut high = 65_536;
while low < high {
let mid = (low + high + 1) / 2;
match self.storage.content.view.0.permit.try_reserve(demand(mid)) {
Ok(permit) => {
drop(permit);
low = mid;
}
Err(_) => high = mid - 1,
}
}
self.storage
.content
.view
.0
.permit
.try_reserve(demand(low))
.unwrap()
}
pub(crate) fn share_for_supervisor(&self) -> Self {
Self {
storage: self.storage.clone(),
task: self.task.clone(),
}
}
pub(crate) fn view(&self) -> ReservedRequestView {
self.storage.content.view.clone()
}
pub fn diagnostic_view(&self) -> ReservedRequestView {
self.view()
}
pub fn retain_database_failure(
&self,
failure: ReservedRequestFailure,
) -> Result<(), ReservedRequestFailure> {
if !self
.storage
.content
.view
.0
.view
.same_view(&failure.source.0.view)
{
return Err(failure);
}
let mut state = self
.storage
.content
.state
.lock()
.unwrap_or_else(|p| p.into_inner());
if state.stage_finished || state.database.is_some() {
return Err(failure);
}
state.database = Some(failure);
Ok(())
}
pub(crate) fn retain(&self, failure: ReservedRequestFailure, cleanup: bool) {
let mut state = self
.storage
.content
.state
.lock()
.unwrap_or_else(|p| p.into_inner());
if cleanup {
state.cleanup.get_or_insert(failure);
} else {
state.primary.get_or_insert(failure);
}
}
pub(crate) fn primary(&self) -> Option<saddle_core::DiagnosticOccurrence> {
let state = self
.storage
.content
.state
.lock()
.unwrap_or_else(|p| p.into_inner());
state
.primary
.as_ref()
.or(state.database.as_ref())
.map(ReservedRequestFailure::occurrence)
}
pub(crate) fn record_stop(&self, stop: crate::profusegw::ProfuseGwScopeStop) {
self.storage
.content
.state
.lock()
.unwrap_or_else(|p| p.into_inner())
.stop
.get_or_insert(stop);
}
pub(crate) fn record_panic(&self) {
self.storage
.content
.state
.lock()
.unwrap_or_else(|p| p.into_inner())
.panicked = true;
}
pub(crate) fn complete(&self) -> ReservedScopeCompletion {
let mut state = self
.storage
.content
.state
.lock()
.unwrap_or_else(|p| p.into_inner());
state.finished = true;
if !state.stage_started {
state.stage_finished = true;
}
drop(state);
retire_clean(&self.task);
ReservedScopeCompletion {
storage: self.storage.clone(),
}
}
pub fn start_database_stage<'a>(
&'a self,
observer: &'a Observer,
output: Option<&'a EmergencyDiagnosticHandle>,
) -> Result<ReservedSupervisedStage<'a>, ()> {
let mut state = self
.storage
.content
.state
.lock()
.unwrap_or_else(|p| p.into_inner());
if state.stage_started || state.finished {
return Err(());
}
state.stage_started = true;
drop(state);
Ok(ReservedSupervisedStage {
stage: self.storage.content.view.start_stage(
observer,
output,
ReservedObservationStage::Database,
),
storage: &self.storage,
task: &self.task,
})
}
}
impl<'a> ReservedSupervisedStage<'a> {
pub fn finish(
self,
completion: ReservedScopeCompletion,
mut facts: RootOutcomeFacts,
) -> Result<DiagnosticSubmission, (Self, ReservedScopeCompletion)> {
if !Arc::ptr_eq(self.storage, &completion.storage) {
return Err((self, completion));
}
let mut state = completion
.storage
.content
.state
.lock()
.unwrap_or_else(|p| p.into_inner());
if !state.finished || state.stage_finished {
drop(state);
return Err((self, completion));
}
if state.cleanup.is_some() {
facts.axes.cleanup = CleanupOutcome::Failed;
}
let (failure, slot) = if state.primary.is_some() {
(state.primary.take(), 0)
} else if state.database.is_some() {
(state.database.take(), 1)
} else {
(state.cleanup.take(), 2)
};
let Self {
stage,
storage,
task,
} = self;
if let Some(failure) = failure {
match stage.finish_failure_retained(failure, facts) {
Ok((failure, submission)) => {
match slot {
0 => state.primary = Some(failure),
1 => state.database = Some(failure),
_ => state.cleanup = Some(failure),
}
state.stage_finished = true;
Ok(submission)
}
Err((stage, failure)) => {
match slot {
0 => state.primary = Some(failure),
1 => state.database = Some(failure),
_ => state.cleanup = Some(failure),
}
drop(state);
Err((
Self {
stage,
storage,
task,
},
completion,
))
}
}
} else {
if !matches!(
facts.axes.operation,
OperationOutcome::Succeeded | OperationOutcome::Rejected
) {
drop(state);
return Err((
Self {
stage,
storage,
task,
},
completion,
));
}
match stage.finish_nonfailure(facts) {
Ok(submission) => {
state.stage_finished = true;
drop(state);
retire_clean(task);
Ok(submission)
}
Err(stage) => {
drop(state);
Err((
Self {
stage,
storage,
task,
},
completion,
))
}
}
}
}
}
fn retire_clean(task: &Retained<TaskStatusStorage>) {
let mut task = task.lock().unwrap_or_else(|p| p.into_inner());
let mut previous: Option<Retained<ScopeStorage>> = None;
let mut current = task.scopes.clone();
while let Some(node) = current {
let mut state = node.content.state.lock().unwrap_or_else(|p| p.into_inner());
let clean = state.finished
&& state.stage_finished
&& state.primary.is_none()
&& state.database.is_none()
&& state.cleanup.is_none();
let next = state.next.clone();
if clean {
state.next = None;
drop(state);
if let Some(previous) = previous.as_ref() {
previous
.content
.state
.lock()
.unwrap_or_else(|p| p.into_inner())
.next = next.clone();
} else {
task.scopes = next.clone();
}
} else {
drop(state);
previous = Some(node);
}
current = next;
}
}
pub(crate) fn has_cleanup(head: &Option<Retained<ScopeStorage>>) -> bool {
let mut next = head.clone();
while let Some(node) = next {
if node
.content
.state
.lock()
.unwrap_or_else(|p| p.into_inner())
.cleanup
.is_some()
{
return true;
}
next = node
.content
.state
.lock()
.unwrap_or_else(|p| p.into_inner())
.next
.clone();
}
false
}
pub(crate) fn project_scopes(
mut head: Option<Retained<ScopeStorage>>,
project: &impl Fn(ReservedRequestFailure) -> PublicRequestFailure,
) -> (Option<PublicRequestFailure>, Option<PublicRequestFailure>) {
let (mut primary, mut cleanup) = (None, None);
while let Some(node) = head {
let mut state = node.content.state.lock().unwrap_or_else(|p| p.into_inner());
head = state.next.take();
state.finished = true;
state.stage_finished = true;
if let Some(failure) = state.database.take() {
primary = Some(project(failure));
}
let mut extra = state.database_more.take();
while let Some(node) = extra {
extra = node.content.next.clone();
if let Some(failure) = node
.content
.failure
.lock()
.unwrap_or_else(|p| p.into_inner())
.take()
{
let public = project(failure);
if primary.is_none() {
primary = Some(public);
}
}
}
if let Some(failure) = state.primary.take() {
primary = Some(project(failure));
}
if let Some(failure) = state.cleanup.take() {
cleanup = Some(project(failure));
}
}
(primary, cleanup)
}
pub fn supervised_scope_layouts() -> [(&'static str, Layout); 8] {
[
(
"scope_operation",
Layout::new::<ReservedScopeOperation<'static>>(),
),
(
"additional_failure_shared_allocation",
shared_prefix::<OperationFailureStorage>().expect("fixed failure layout"),
),
("scope_storage", Layout::new::<ScopeStorage>()),
(
"scope_shared_allocation",
shared_prefix::<ScopeStorage>().expect("fixed scope layout"),
),
(
"scope_observation",
Layout::new::<ReservedScopeObservation>(),
),
(
"task_scope_head",
Layout::new::<Option<Retained<ScopeStorage>>>(),
),
("scope_completion", Layout::new::<ReservedScopeCompletion>()),
(
"supervised_stage",
Layout::new::<ReservedSupervisedStage<'static>>(),
),
]
}