use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use uuid::Uuid;
use crate::capability::{Capabilities, OwnershipIdentity};
use crate::config::{DnsConfig, DnsScope, validate_against};
use crate::error::{ConflictReason, Error, Result};
use crate::fault::{CrashSignal, FaultAction, FaultHook, TxPoint};
use crate::fsutil::ensure_private_dir;
use crate::interface::InterfaceInfo;
use crate::journal::{JournalRecord, JournalStore, Phase, SCHEMA_VERSION};
use crate::lease::{Lease, LiveRecord};
use crate::normalize::NormalizedConfig;
use crate::ownership::{ResourceId, ResourceLockManager};
use crate::platform::{
Backend, MutationAttempt, OwnershipProof, PlatformSnapshot, ResourceIdentity, ResourceStatus,
VerifiedMutation, select_default_backend,
};
use crate::reconciliation::Reconciler;
use crate::watch::SuppressionRegistry;
use crate::watch::{WatchCallback, WatchHandle};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum ConflictPolicy {
#[default]
Cooperative,
Enforce,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum RecoveryOutcome {
Restored {
resource: ResourceId,
lease_id: Uuid,
},
JournalCleared {
resource: ResourceId,
lease_id: Uuid,
},
Gone {
resource: ResourceId,
lease_id: Uuid,
},
Replaced {
resource: ResourceId,
lease_id: Uuid,
},
IdentityMismatch {
resource: ResourceId,
lease_id: Uuid,
},
Failed {
resource: ResourceId,
lease_id: Uuid,
detail: String,
},
ExternalConflict {
resource: ResourceId,
lease_id: Uuid,
},
Busy {
resource: ResourceId,
},
}
pub(crate) struct Inner {
pub(crate) owner: String,
pub(crate) backend: Arc<dyn Backend>,
pub(crate) locks: ResourceLockManager,
pub(crate) journal: JournalStore,
pub(crate) conflict_policy: ConflictPolicy,
pub(crate) hook: Mutex<Option<Arc<dyn FaultHook>>>,
pub(crate) suppressions: Arc<SuppressionRegistry>,
pub(crate) active: Mutex<HashMap<ResourceId, Arc<Mutex<LiveRecord>>>>,
pub(crate) lease_tokens: Mutex<HashMap<ResourceId, Arc<Mutex<()>>>>,
#[allow(dead_code)]
pub(crate) reconciler: Reconciler,
pub(crate) enforce: Mutex<EnforceState>,
}
#[derive(Default)]
pub(crate) struct EnforceState {
refs: usize,
handle: Option<WatchHandle>,
feed: Option<std::sync::mpsc::Sender<ResourceId>>,
parked: bool,
}
impl std::fmt::Debug for EnforceState {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("EnforceState")
.field("refs", &self.refs)
.field("watching", &self.handle.is_some())
.finish()
}
}
const COALESCE_WINDOW: Duration = Duration::from_millis(50);
impl Inner {
pub(crate) fn ensure_enforce_watch(self: &Arc<Self>) -> Result<()> {
if self.conflict_policy != ConflictPolicy::Enforce {
return Ok(());
}
if !self.backend.capabilities().watch {
return Err(Error::unsupported(
self.backend.kind(),
"ConflictPolicy::Enforce requires change notifications, which this backend does not support",
));
}
let mut enforce = self
.enforce
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
if enforce.refs == 0 {
debug_assert!(enforce.handle.is_none() && enforce.feed.is_none());
let feed = crate::reconciliation::spawn_reconciler(Arc::clone(self))?;
let feed_clone = feed.clone();
let callback: WatchCallback = Arc::new(move |event| {
let _ = feed_clone.send(event.resource().clone());
});
match self.backend.start_watch(callback) {
Ok(handle) => {
enforce.handle = Some(handle);
enforce.feed = Some(feed);
enforce.parked = false;
}
Err(error) => {
drop(feed);
return Err(error);
}
}
}
enforce.refs += 1;
Ok(())
}
pub(crate) fn release_enforce_watch(&self) {
if self.conflict_policy != ConflictPolicy::Enforce {
return;
}
let mut enforce = self
.enforce
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
if enforce.refs == 0 {
return;
}
enforce.refs -= 1;
if enforce.refs == 0 {
enforce.handle = None;
enforce.feed = None;
enforce.parked = false;
}
}
pub(crate) fn enforce_feed(&self) -> Option<std::sync::mpsc::Sender<ResourceId>> {
self.enforce
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.feed
.clone()
}
#[cfg(feature = "test-util")]
#[allow(dead_code)]
pub(crate) fn enforce_refs(&self) -> usize {
self.enforce
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.refs
}
pub(crate) fn enforce_parked(&self) -> bool {
self.enforce
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.parked
}
#[cfg(feature = "test-util")]
#[allow(dead_code)]
pub(crate) fn suspend_enforce_watch(&self) {
{
let mut enforce = self
.enforce
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
enforce.handle = None;
enforce.feed = None;
enforce.parked = true;
}
self.reconciler.clear();
}
pub(crate) fn rescan_enforce(&self) {
let Some(feed) = self.enforce_feed() else {
return;
};
let resources: Vec<ResourceId> = self
.active
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.keys()
.cloned()
.collect();
for resource in resources {
let _ = feed.send(resource);
}
}
#[cfg(feature = "test-util")]
#[allow(dead_code)]
pub(crate) fn enforce_watching(&self) -> bool {
self.enforce
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.handle
.is_some()
}
}
pub(crate) struct MutatePoints {
apply: TxPoint,
readback: TxPoint,
verify: TxPoint,
}
pub(crate) const INITIAL_POINTS: MutatePoints = MutatePoints {
apply: TxPoint::AfterApply,
readback: TxPoint::AfterReadback,
verify: TxPoint::AfterVerify,
};
const UPDATE_POINTS: MutatePoints = MutatePoints {
apply: TxPoint::AfterUpdateApply,
readback: TxPoint::AfterUpdateReadback,
verify: TxPoint::AfterUpdateVerify,
};
pub(crate) struct MutationResidue {
pub(crate) leftover: Option<OwnershipProof>,
restored: Option<PlatformSnapshot>,
}
impl MutationResidue {
pub(crate) fn new() -> Self {
Self {
leftover: None,
restored: None,
}
}
}
pub(crate) struct PreparedLease {
lease_id: Uuid,
records: Vec<JournalRecord>,
verified: Vec<Option<OwnershipProof>>,
was_noop: bool,
}
enum RecoverBlock {
Journal(Error),
Conflict(String),
}
impl Inner {
pub(crate) fn fire(&self, point: TxPoint) -> Result<()> {
let hook = self
.hook
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.clone();
if let Some(hook) = hook {
match hook.on_point(point) {
FaultAction::Continue => {}
FaultAction::Crash => std::panic::panic_any(CrashSignal),
FaultAction::Fail(message) => {
return Err(Error::platform(
self.backend.kind(),
format_args!("injected transaction failure: {message}"),
));
}
}
}
Ok(())
}
pub(crate) fn mutate_and_verify(
&self,
identity: &ResourceIdentity,
expected_current: &PlatformSnapshot,
plan: &NormalizedConfig,
rollback_to: Option<&PlatformSnapshot>,
points: MutatePoints,
residue: &mut MutationResidue,
) -> Result<VerifiedMutation> {
let resource = &identity.resource;
residue.leftover = None;
residue.restored = None;
self.suppressions.suppress(resource);
match self.apply_attempt(identity, expected_current, plan) {
MutationAttempt::Rejected { error } => Err(error),
MutationAttempt::Indeterminate { error, produced } => {
match self.rollback_proven(resource, identity, produced.as_ref(), rollback_to) {
Some(identity) => residue.restored = identity,
None => residue.leftover = produced.map(OwnershipProof::issued),
}
Err(error)
}
MutationAttempt::Performed { produced } => {
let fail = |this: &Self,
residue: &mut MutationResidue,
produced: Option<PlatformSnapshot>,
error: Error| {
match this.rollback_proven(resource, identity, produced.as_ref(), rollback_to) {
Some(identity) => residue.restored = identity,
None => residue.leftover = produced.map(OwnershipProof::issued),
}
Err(error)
};
if let Err(error) = self.fire(points.apply) {
return fail(self, residue, produced, error);
}
match self.backend.resource_status(identity) {
Ok(ResourceStatus::Same) => {}
Ok(status) => {
return fail(
self,
residue,
produced,
Error::ResourceIdentity {
backend: self.backend.kind(),
resource: resource.clone(),
message: format!(
"resource incarnation became {status:?} during mutation"
),
},
);
}
Err(error) => return fail(self, residue, produced, error),
}
match self.backend.readback(resource) {
Ok(actual) => {
if let Err(error) = self.fire(points.readback) {
return fail(self, residue, produced, error);
}
if !self.backend.matches_desired(&actual, plan) {
return fail(
self,
residue,
produced,
Error::VerificationFailed {
resource: resource.clone(),
detail: "the state read back from the system does not match the desired configuration"
.to_string(),
},
);
}
match self.backend.resource_status(identity) {
Ok(ResourceStatus::Same) => {}
Ok(status) => {
return fail(
self,
residue,
produced,
Error::ResourceIdentity {
backend: self.backend.kind(),
resource: resource.clone(),
message: format!(
"resource incarnation became {status:?} during verification"
),
},
);
}
Err(error) => return fail(self, residue, produced, error),
}
if let Err(error) = self.fire(points.verify) {
return fail(self, residue, produced, error);
}
match self.backend.ownership_identity() {
OwnershipIdentity::Durable => {
let Some(produced) = produced else {
return fail(
self,
residue,
None,
Error::VerificationFailed {
resource: resource.clone(),
detail: "the mutation issued no ownership proof"
.to_string(),
},
);
};
if !self.backend.proves_current(&produced, &actual) {
return fail(
self,
residue,
Some(produced),
Error::ExternalModification {
resource: resource.clone(),
detail: "the state read back does not carry the identity of our mutation"
.to_string(),
},
);
}
Ok(VerifiedMutation {
proof: Some(OwnershipProof::issued(produced)),
observed: actual,
})
}
OwnershipIdentity::BestEffort => Ok(VerifiedMutation {
proof: produced.map(OwnershipProof::issued),
observed: actual,
}),
}
}
Err(error) => fail(self, residue, produced, error),
}
}
}
}
fn apply_attempt(
&self,
identity: &ResourceIdentity,
expected: &PlatformSnapshot,
plan: &NormalizedConfig,
) -> MutationAttempt {
self.backend.apply_bound(identity, expected, plan)
}
fn restore_if_current(
&self,
_resource: &ResourceId,
identity: &ResourceIdentity,
expected: &PlatformSnapshot,
target: &PlatformSnapshot,
) -> Result<Option<PlatformSnapshot>> {
match self.backend.restore_bound(identity, expected, target) {
MutationAttempt::Performed { produced } => Ok(produced),
MutationAttempt::Rejected { error } | MutationAttempt::Indeterminate { error, .. } => {
Err(error)
}
}
}
#[allow(unused_variables)]
fn rollback_proven(
&self,
resource: &ResourceId,
identity: &ResourceIdentity,
proof: Option<&PlatformSnapshot>,
target: Option<&PlatformSnapshot>,
) -> Option<Option<PlatformSnapshot>> {
let (Some(proof), Some(target)) = (proof, target) else {
return None;
};
if self.backend.equivalent(proof, target) {
return Some(Some(proof.clone()));
}
let produced = match self.restore_if_current(resource, identity, proof, target) {
Ok(produced) => produced,
Err(error) => {
osdns_warn!(
resource = %resource,
error = %error,
"rollback could not restore the previous state; the journal record was kept for later recovery"
);
return None;
}
};
match self.backend.readback(resource) {
Ok(now) if self.backend.equivalent(&now, target) => Some(produced),
Ok(_) => None,
Err(error) => {
osdns_warn!(
resource = %resource,
error = %error,
"rollback could not read back the restored state; the journal record was kept for later recovery"
);
None
}
}
}
pub(crate) fn transact_with_locks(
&self,
resources: Vec<ResourceId>,
plan: &NormalizedConfig,
befores: Vec<PlatformSnapshot>,
identities: Vec<ResourceIdentity>,
) -> Result<PreparedLease> {
let lease_id = Uuid::new_v4();
let was_noop = resources
.iter()
.zip(&befores)
.all(|(_resource, before)| self.backend.matches_desired(before, plan));
if was_noop {
let records: Vec<JournalRecord> = resources
.into_iter()
.zip(befores)
.zip(identities)
.map(|((resource, before), identity)| JournalRecord {
schema_version: SCHEMA_VERSION,
owner: self.owner.clone(),
lease_id,
resource,
backend: self.backend.kind(),
identity,
phase: Phase::Applied,
before: before.clone(),
desired: plan.clone(),
applied: Some(before),
})
.collect();
for record in &records {
self.journal.write(record)?;
}
self.fire(TxPoint::AfterApplied)?;
let n = records.len();
return Ok(PreparedLease {
lease_id,
records,
verified: vec![None; n],
was_noop: true,
});
}
let mut records: Vec<JournalRecord> = resources
.into_iter()
.zip(befores)
.zip(identities)
.map(|((resource, before), identity)| JournalRecord {
schema_version: SCHEMA_VERSION,
owner: self.owner.clone(),
lease_id,
resource,
backend: self.backend.kind(),
identity,
phase: Phase::Prepared,
before,
desired: plan.clone(),
applied: None,
})
.collect();
for record in &records {
self.journal.write(record)?;
}
self.fire(TxPoint::AfterPrepared)?;
let mut mutations: Vec<VerifiedMutation> = Vec::new();
for index in 0..records.len() {
let expected = records[index].before.clone();
let mut residue = MutationResidue::new();
match self.mutate_and_verify(
&records[index].identity,
&expected,
plan,
Some(&records[index].before),
INITIAL_POINTS,
&mut residue,
) {
Ok(mutation) => mutations.push(mutation),
Err(error) => {
for (record, mutation) in records[..index].iter().zip(&mutations) {
if self
.rollback_proven(
&record.resource,
&record.identity,
Some(&mutation.persist()),
Some(&record.before),
)
.is_some()
{
let _ = self.journal.remove(&record.lease_id, &record.resource);
}
}
let failed = &records[index];
if residue.leftover.is_none()
&& let Ok(now) = self.backend.readback(&failed.resource)
&& self.backend.equivalent(&now, &failed.before)
{
let _ = self.journal.remove(&failed.lease_id, &failed.resource);
}
for record in &records[index + 1..] {
let _ = self.journal.remove(&record.lease_id, &record.resource);
}
return Err(error);
}
}
}
let mut verified = vec![None; records.len()];
let mut all_written = true;
for (index, record) in records.iter_mut().enumerate() {
record.phase = Phase::Applied;
record.applied = Some(mutations[index].persist());
if self.journal.write(record).is_err() {
verified[index] = mutations[index]
.proof
.clone()
.or_else(|| record.applied.take().map(OwnershipProof::issued));
record.applied = None;
record.phase = Phase::Prepared;
all_written = false;
}
}
if all_written {
self.fire(TxPoint::AfterApplied)?;
}
Ok(PreparedLease {
lease_id,
records,
verified,
was_noop: false,
})
}
pub(crate) fn transact_update(
&self,
live: &[Arc<Mutex<LiveRecord>>],
plan: &NormalizedConfig,
) -> Result<()> {
self.fire(TxPoint::AfterUpdateResolve)?;
let mut olds: Vec<JournalRecord> = Vec::with_capacity(live.len());
let mut applieds: Vec<PlatformSnapshot> = Vec::with_capacity(live.len());
for record in live {
let mut guard = record
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
self.finalize_live(&mut guard, None)?;
let applied = guard.record.applied.clone().ok_or_else(|| {
Error::ExternalModification {
resource: guard.record.resource.clone(),
detail: "the lease record carries no verified applied state; refusing to overwrite indeterminate state"
.to_string(),
}
})?;
olds.push(guard.record.clone());
applieds.push(applied);
}
for index in 0..live.len() {
let resource = olds[index].resource.clone();
let current = self.backend.readback(&resource)?;
self.fire(TxPoint::AfterUpdateCapture)?;
if !self.backend.owns_current(&applieds[index], ¤t) {
return Err(Error::ExternalModification {
resource,
detail: "the current state no longer matches the state applied by this lease"
.to_string(),
});
}
}
if applieds
.iter()
.all(|applied| self.backend.matches_desired(applied, plan))
{
self.fire(TxPoint::AfterUpdateNoopCheck)?;
return Ok(());
}
for record in live {
let mut guard = record
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
guard.record.desired = plan.clone();
guard.record.phase = Phase::Prepared;
if let Err(error) = self.journal.write(&guard.record) {
drop(guard);
for (old, live_record) in olds.iter().zip(live.iter()) {
let mut guard = live_record
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
guard.record = old.clone();
let _ = self.journal.write(&guard.record);
}
return Err(error);
}
}
self.fire(TxPoint::AfterUpdatePrepared)?;
let mut mutations: Vec<Option<VerifiedMutation>> = vec![None; live.len()];
for index in 0..live.len() {
let expected = applieds[index].clone();
let mut residue = MutationResidue::new();
match self.mutate_and_verify(
&olds[index].identity,
&expected,
plan,
Some(&applieds[index]),
UPDATE_POINTS,
&mut residue,
) {
Ok(mutation) => mutations[index] = Some(mutation),
Err(error) => {
let mut restored_identity: Vec<Option<PlatformSnapshot>> =
vec![None; live.len()];
for rollback_index in 0..index {
if let Some(mutation) = &mutations[rollback_index]
&& let Some(Some(identity)) = self.rollback_proven(
&olds[rollback_index].resource,
&olds[rollback_index].identity,
Some(&mutation.persist()),
Some(&applieds[rollback_index]),
)
{
restored_identity[rollback_index] = Some(identity);
}
}
self.fire(TxPoint::AfterUpdateVerify).ok();
for (old_index, (old, live_record)) in olds.iter().zip(live.iter()).enumerate()
{
let mut guard = live_record
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
if old_index == index && residue.leftover.is_some() {
guard.verified = residue.leftover.take();
guard.record.phase = Phase::Prepared;
guard.record.applied = None;
let _ = self.journal.write(&guard.record);
} else {
guard.record = old.clone();
if old_index == index {
if let Some(identity) = residue.restored.take() {
guard.record.applied = Some(identity);
}
} else if let Some(identity) = restored_identity[old_index].take() {
guard.record.applied = Some(identity);
}
guard.verified = None;
let _ = self.journal.write(&guard.record);
}
}
return Err(error);
}
}
}
let mut write_error = None;
for (index, record) in live.iter().enumerate() {
let mut guard = record
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let mutation = mutations[index].as_ref().expect("mutation succeeded");
let persisted = mutation.persist();
guard.record.phase = Phase::Applied;
guard.record.applied = Some(persisted.clone());
if let Err(error) = self.journal.write(&guard.record) {
guard.verified = mutation
.proof
.clone()
.or(Some(OwnershipProof::issued(persisted)));
guard.record.applied = None;
guard.record.phase = Phase::Prepared;
if write_error.is_none() {
write_error = Some(error);
}
} else {
guard.verified = None;
}
}
if let Some(error) = write_error {
return Err(error);
}
self.fire(TxPoint::AfterUpdateApplied)?;
Ok(())
}
pub(crate) fn finalize_live(
&self,
live: &mut LiveRecord,
current: Option<&PlatformSnapshot>,
) -> Result<()> {
if live.record.phase == Phase::Applied && live.record.applied.is_some() {
live.verified = None;
return Ok(());
}
let Some(proof) = live.verified.clone() else {
return Err(Error::ExternalModification {
resource: live.record.resource.clone(),
detail: "the lease record carries no verified applied state; refusing to overwrite indeterminate state"
.to_string(),
});
};
let owned;
let current = match current {
Some(current) => current,
None => {
owned = self.backend.readback(&live.record.resource)?;
&owned
}
};
if !self.backend.owns_current(proof.as_snapshot(), current) {
return Err(Error::ExternalModification {
resource: live.record.resource.clone(),
detail: "the retained mutation proof no longer names the current state".to_string(),
});
}
live.record.applied = Some(proof.into_snapshot());
live.record.phase = Phase::Applied;
self.journal.write(&live.record)?;
live.verified = None;
Ok(())
}
pub(crate) fn restore_lease_state(&self, record: &JournalRecord) -> Result<()> {
let resource = &record.resource;
match self.backend.resource_status(&record.identity)? {
ResourceStatus::Gone | ResourceStatus::Replaced => {
self.journal.remove(&record.lease_id, resource)?;
return Ok(());
}
ResourceStatus::Ambiguous => {
return Err(Error::ResourceIdentity {
backend: self.backend.kind(),
resource: resource.clone(),
message: "the backend cannot prove that the current target is the leased native resource incarnation".to_string(),
});
}
ResourceStatus::Same => {}
}
self.suppressions.suppress(resource);
let current = match self.backend.readback(resource) {
Ok(current) => current,
Err(Error::ResourceGone { .. }) => {
self.journal.remove(&record.lease_id, resource)?;
return Ok(());
}
Err(error) => return Err(error),
};
self.fire(TxPoint::AfterRestoreReadback)?;
if self.backend.equivalent(¤t, &record.before) {
self.journal.remove(&record.lease_id, resource)?;
self.fire(TxPoint::AfterRestoreJournal)?;
return Ok(());
}
let applied = record.applied.as_ref().ok_or_else(|| Error::ExternalModification {
resource: resource.clone(),
detail: "the lease record carries no verified applied state; refusing to overwrite indeterminate state"
.to_string(),
})?;
if !self.backend.owns_current(applied, ¤t) {
return Err(Error::ExternalModification {
resource: resource.clone(),
detail: "the current state is neither the state applied by this lease nor the original state"
.to_string(),
});
}
self.restore_if_current(resource, &record.identity, applied, &record.before)?;
self.fire(TxPoint::AfterRestoreRestore)?;
let now = self.backend.readback(resource)?;
if !self.backend.equivalent(&now, &record.before) {
return Err(Error::VerificationFailed {
resource: resource.clone(),
detail: "the restored state failed read-back verification".to_string(),
});
}
self.journal.remove(&record.lease_id, resource)?;
self.fire(TxPoint::AfterRestoreJournal)?;
Ok(())
}
#[allow(unused_variables)]
pub(crate) fn best_effort_restore(&self, record: &JournalRecord) {
if let Err(error) = self.restore_lease_state(record) {
osdns_warn!(
owner = %self.owner,
resource = %record.resource,
error = %error,
"best-effort restore on lease drop failed; the journal record was kept for later recovery"
);
}
}
fn recover_record(&self, record: JournalRecord) -> Result<RecoveryOutcome> {
let resource = record.resource.clone();
match self.backend.resource_status(&record.identity)? {
ResourceStatus::Gone => {
self.journal.remove(&record.lease_id, &resource)?;
return Ok(RecoveryOutcome::Gone {
resource,
lease_id: record.lease_id,
});
}
ResourceStatus::Replaced => {
self.journal.remove(&record.lease_id, &resource)?;
return Ok(RecoveryOutcome::Replaced {
resource,
lease_id: record.lease_id,
});
}
ResourceStatus::Ambiguous => {
return Ok(RecoveryOutcome::IdentityMismatch {
resource,
lease_id: record.lease_id,
});
}
ResourceStatus::Same => {}
}
let current = match self.backend.capture(&resource) {
Ok(current) => current,
Err(Error::ResourceGone { .. }) => {
self.journal.remove(&record.lease_id, &resource)?;
return Ok(RecoveryOutcome::Gone {
resource,
lease_id: record.lease_id,
});
}
Err(error) => return Err(error),
};
self.fire(TxPoint::AfterRecoveryReadback)?;
if self.backend.equivalent(¤t, &record.before) {
self.journal.remove(&record.lease_id, &resource)?;
self.fire(TxPoint::AfterRecoveryJournal)?;
return Ok(RecoveryOutcome::JournalCleared {
resource,
lease_id: record.lease_id,
});
}
let owned = record
.applied
.as_ref()
.is_some_and(|applied| self.backend.owns_current(applied, ¤t));
if owned {
self.suppressions.suppress(&resource);
let applied = record.applied.as_ref().expect("applied snapshot");
if let Err(error) =
self.restore_if_current(&resource, &record.identity, applied, &record.before)
{
if error.is_external_modification() {
return Ok(RecoveryOutcome::ExternalConflict {
resource,
lease_id: record.lease_id,
});
}
return Err(error);
}
self.fire(TxPoint::AfterRecoveryRestore)?;
let now = self.backend.readback(&resource)?;
if !self.backend.equivalent(&now, &record.before) {
return Err(Error::VerificationFailed {
resource,
detail: "the recovery restore did not read back as the original state"
.to_string(),
});
}
self.journal.remove(&record.lease_id, &resource)?;
self.fire(TxPoint::AfterRecoveryJournal)?;
return Ok(RecoveryOutcome::Restored {
resource,
lease_id: record.lease_id,
});
}
Ok(RecoveryOutcome::ExternalConflict {
resource,
lease_id: record.lease_id,
})
}
fn recover_for_resource(&self, resource: &ResourceId) -> std::result::Result<(), RecoverBlock> {
let records = self
.journal
.records_for(resource)
.map_err(RecoverBlock::Journal)?;
let mut conflict = None;
for record in records {
match self.recover_record(record) {
Ok(RecoveryOutcome::ExternalConflict { .. }) => {
conflict = Some(
"the current state matches neither the journal's applied state nor its original state"
.to_string(),
);
}
Ok(RecoveryOutcome::IdentityMismatch { .. } | RecoveryOutcome::Failed { .. }) => {
conflict = Some(
"the recorded native resource incarnation cannot be recovered safely"
.to_string(),
);
}
Ok(_) => {}
Err(error @ Error::JournalCorrupt(_)) => {
return Err(RecoverBlock::Journal(error));
}
Err(error) => {
return Err(RecoverBlock::Conflict(error.to_string()));
}
}
}
match conflict {
Some(detail) => Err(RecoverBlock::Conflict(detail)),
None => Ok(()),
}
}
pub(crate) fn recover_stale(&self) -> Result<Vec<RecoveryOutcome>> {
let records = self.journal.records()?;
let mut outcomes = Vec::new();
for record in records {
let resource = record.resource.clone();
let lease_id = record.lease_id;
match self.locks.try_acquire(&resource) {
Ok(Some(lock)) => {
let outcome = self.recover_record(record).unwrap_or_else(|error| {
RecoveryOutcome::Failed {
resource: resource.clone(),
lease_id,
detail: error.to_string(),
}
});
drop(lock);
outcomes.push(outcome);
}
Ok(None) => outcomes.push(RecoveryOutcome::Busy { resource }),
Err(Error::Conflict { .. }) => {
outcomes.push(RecoveryOutcome::Busy { resource });
}
Err(error) => outcomes.push(RecoveryOutcome::Failed {
resource,
lease_id,
detail: error.to_string(),
}),
}
}
Ok(outcomes)
}
pub(crate) fn abandon_journal(&self, resource: &ResourceId) -> Result<()> {
let lock = self.locks.acquire(resource)?;
let records = self.journal.records_for(resource)?;
let mut result = Ok(());
for record in records {
if let Err(error) = self.journal.remove(&record.lease_id, resource) {
result = Err(error);
break;
}
}
drop(lock);
result
}
}
#[derive(Clone)]
pub struct DnsManager {
inner: Arc<Inner>,
}
impl DnsManager {
pub(crate) fn from_inner(inner: Arc<Inner>) -> Self {
Self { inner }
}
pub fn builder() -> DnsManagerBuilder {
DnsManagerBuilder::new()
}
pub fn owner(&self) -> &str {
&self.inner.owner
}
pub fn conflict_policy(&self) -> ConflictPolicy {
self.inner.conflict_policy
}
pub fn capabilities(&self) -> Result<Capabilities> {
Ok(self.inner.backend.capabilities())
}
pub fn interfaces(&self) -> Result<Vec<InterfaceInfo>> {
self.inner.backend.list_interfaces()
}
pub fn snapshot(&self, scope: &DnsScope) -> Result<DnsConfig> {
let resources = self
.inner
.backend
.resolve_resources(scope, &NormalizedConfig::default())?;
let resource = resources.first().ok_or_else(|| {
Error::invalid_config("the backend resolved the scope to no resources")
})?;
let snapshot = self.inner.backend.capture(resource)?;
self.inner.backend.public_state(&snapshot, scope)
}
pub fn validate(&self, config: &DnsConfig) -> Result<()> {
let caps = self.inner.backend.capabilities();
let plan = validate_against(config, &caps)?;
self.inner.backend.validate_plan(config.scope(), &plan)?;
Ok(())
}
pub fn apply(&self, config: &DnsConfig) -> Result<Lease> {
let caps = self.inner.backend.capabilities();
let plan = validate_against(config, &caps)?;
self.inner.backend.validate_plan(config.scope(), &plan)?;
self.inner.fire(TxPoint::AfterValidate)?;
let resources = self
.inner
.backend
.resolve_resources(config.scope(), &plan)?;
self.inner.fire(TxPoint::AfterResolve)?;
let locks = self.inner.locks.acquire_all(&resources)?;
self.inner.fire(TxPoint::AfterLock)?;
for resource in &resources {
match self.inner.recover_for_resource(resource) {
Ok(()) => {}
Err(RecoverBlock::Journal(error)) => return Err(error),
Err(RecoverBlock::Conflict(detail)) => {
return Err(Error::Conflict {
resource: resource.clone(),
reason: ConflictReason::StaleJournalUnresolved { detail },
});
}
}
}
self.inner.fire(TxPoint::AfterRecovery)?;
let mut befores = Vec::with_capacity(resources.len());
let mut identities = Vec::with_capacity(resources.len());
for resource in &resources {
let observation = self.inner.backend.observe(resource)?;
identities.push(observation.identity);
befores.push(observation.snapshot);
self.inner.fire(TxPoint::AfterCapture)?;
}
self.inner.fire(TxPoint::AfterNoopDecision)?;
match self
.inner
.transact_with_locks(resources, &plan, befores, identities)
{
Ok(PreparedLease {
lease_id,
records,
verified,
was_noop,
}) => {
let lease = Lease::new_owned(
self.inner.clone(),
lease_id,
records,
verified,
locks,
was_noop,
);
if let Err(error) = self.inner.ensure_enforce_watch() {
let _ = lease.restore();
return Err(error);
}
self.inner.rescan_enforce();
Ok(lease)
}
Err(error) => Err(error),
}
}
pub fn watch(&self, callback: WatchCallback) -> Result<WatchHandle> {
let feed = if self.inner.conflict_policy == ConflictPolicy::Enforce {
match self.inner.enforce_feed() {
Some(existing) => Some(existing),
None => Some(crate::reconciliation::spawn_reconciler(self.inner.clone())?),
}
} else {
None
};
let coalescer =
crate::watch::spawn_coalescer(self.inner.backend.kind(), callback, COALESCE_WINDOW)?;
let coalesced = coalescer.callback();
let suppressions = Arc::clone(&self.inner.suppressions);
let filtered: WatchCallback = Arc::new(move |event| {
if let Some(feed) = &feed {
let _ = feed.send(event.resource().clone());
}
if suppressions.is_suppressed(event.resource()) {
return;
}
coalesced(event);
});
let native = self.inner.backend.start_watch(filtered)?;
let flag = Arc::new(std::sync::atomic::AtomicBool::new(false));
Ok(WatchHandle::new(flag, move || {
native.stop();
coalescer.stop();
}))
}
pub fn flush_cache(&self) -> Result<()> {
self.inner.backend.flush_cache()
}
pub fn recover_stale(&self) -> Result<Vec<RecoveryOutcome>> {
self.inner.recover_stale()
}
pub fn abandon_journal(&self, resource: &ResourceId) -> Result<()> {
self.inner.abandon_journal(resource)
}
#[cfg(feature = "test-util")]
pub fn install_fault_injector(&self, injector: Arc<crate::testing::FaultInjector>) {
let hook: Arc<dyn FaultHook> = injector;
*self
.inner
.hook
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(hook);
}
#[cfg(feature = "test-util")]
pub fn set_journal_fail_writes(&self, fail: bool) {
self.inner.journal.set_fail_writes(fail);
}
#[cfg(feature = "test-util")]
pub fn set_journal_fail_writes_after(&self, skip: u32) {
self.inner.journal.set_fail_writes_after(skip);
}
#[cfg(feature = "test-util")]
pub fn set_journal_fail_removes(&self, fail: bool) {
self.inner.journal.set_fail_removes(fail);
}
#[cfg(feature = "test-util")]
pub fn debug_enforce_refs(&self) -> usize {
self.inner.enforce_refs()
}
#[cfg(feature = "test-util")]
pub fn debug_enforce_watching(&self) -> bool {
self.inner.enforce_watching()
}
#[cfg(feature = "test-util")]
pub fn suspend_enforce_background(&self) {
self.inner.suspend_enforce_watch();
}
#[cfg(feature = "test-util")]
pub fn debug_reconcile(&self, resource: &str) -> Result<crate::testing::DebugReconcile> {
use crate::reconciliation::ReconcileOutcome;
let resource: ResourceId = resource.parse().map_err(|e| {
Error::invalid_config(format_args!("invalid resource id {resource:?}: {e}"))
})?;
Ok(
match self
.inner
.reconcile_resource(&resource, &self.inner.reconciler)
{
ReconcileOutcome::NoActiveLease => crate::testing::DebugReconcile::NotOwned,
ReconcileOutcome::IdentityAmbiguous => {
crate::testing::DebugReconcile::IdentityAmbiguous
}
ReconcileOutcome::StillOurs => crate::testing::DebugReconcile::StillOurs,
ReconcileOutcome::Rebased => crate::testing::DebugReconcile::Rebased,
ReconcileOutcome::Deferred => crate::testing::DebugReconcile::Deferred,
ReconcileOutcome::Failed => crate::testing::DebugReconcile::Failed,
},
)
}
#[cfg(feature = "test-util")]
pub fn debug_reconcile_pending(&self, resource: &str) -> Result<bool> {
let resource: ResourceId = resource.parse().map_err(|e| {
Error::invalid_config(format_args!("invalid resource id {resource:?}: {e}"))
})?;
Ok(self.inner.reconciler.is_pending(&resource))
}
}
impl std::fmt::Debug for DnsManager {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DnsManager")
.field("owner", &self.inner.owner)
.field("conflict_policy", &self.inner.conflict_policy)
.finish_non_exhaustive()
}
}
pub struct DnsManagerBuilder {
owner: Option<String>,
state_dir: Option<PathBuf>,
lock_timeout: Duration,
conflict_policy: ConflictPolicy,
}
impl DnsManagerBuilder {
pub(crate) fn new() -> Self {
Self {
owner: None,
state_dir: None,
lock_timeout: Duration::from_secs(30),
conflict_policy: ConflictPolicy::default(),
}
}
pub fn owner(mut self, owner: impl Into<String>) -> Self {
self.owner = Some(owner.into());
self
}
pub fn state_dir(mut self, dir: impl Into<PathBuf>) -> Self {
self.state_dir = Some(dir.into());
self
}
pub fn lock_timeout(mut self, timeout: Duration) -> Self {
self.lock_timeout = timeout;
self
}
pub fn conflict_policy(mut self, policy: ConflictPolicy) -> Self {
self.conflict_policy = policy;
self
}
pub fn build(self) -> Result<DnsManager> {
let owner = self
.owner
.ok_or_else(|| Error::invalid_config("an owner identifier is required"))?;
validate_owner(&owner)?;
if self.lock_timeout.is_zero() {
return Err(Error::invalid_config(
"lock_timeout must be greater than zero",
));
}
let state_dir = match self.state_dir {
Some(dir) => dir,
None => default_state_dir()?,
};
ensure_private_dir(&state_dir)?;
let global_lock_dir = global_lock_root()?.join("locks");
let locks = ResourceLockManager::new(global_lock_dir, self.lock_timeout);
let journal = JournalStore::open(state_dir.join("journal"))?;
let backend = select_default_backend(&owner)?;
if self.conflict_policy == ConflictPolicy::Enforce && !backend.capabilities().watch {
return Err(Error::unsupported(
backend.kind(),
"ConflictPolicy::Enforce requires change notifications, which this backend does not support",
));
}
Ok(DnsManager::from_inner(Arc::new(Inner {
owner,
backend,
locks,
journal,
conflict_policy: self.conflict_policy,
hook: Mutex::new(None),
suppressions: Arc::new(SuppressionRegistry::new()),
active: Mutex::new(HashMap::new()),
lease_tokens: Mutex::new(HashMap::new()),
reconciler: Reconciler::default(),
enforce: Mutex::new(EnforceState::default()),
})))
}
}
impl Default for DnsManagerBuilder {
fn default() -> Self {
Self::new()
}
}
fn validate_owner(owner: &str) -> Result<()> {
if owner.is_empty() || owner.len() > 255 {
return Err(Error::invalid_config(
"owner identifier must be 1-255 characters",
));
}
for c in owner.chars() {
if !(c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_')) {
return Err(Error::invalid_config(format_args!(
"owner identifier {owner:?} contains invalid character {c:?}"
)));
}
}
Ok(())
}
#[cfg(target_os = "windows")]
fn default_state_dir() -> Result<PathBuf> {
crate::platform::windows::programdata::program_data_dir().map(|dir| dir.join("osdns"))
}
#[cfg(target_os = "macos")]
fn default_state_dir() -> Result<PathBuf> {
Ok(PathBuf::from("/Library/Application Support/osdns"))
}
#[cfg(not(any(target_os = "windows", target_os = "macos")))]
fn default_state_dir() -> Result<PathBuf> {
Ok(PathBuf::from("/var/lib/osdns"))
}
pub(crate) fn global_lock_root() -> Result<PathBuf> {
default_state_dir()
}
#[cfg(all(test, target_os = "windows"))]
mod windows_lock_dir_tests {
use super::global_lock_root;
#[test]
fn global_lock_root_ignores_programdata_and_localappdata() {
let original_programdata = std::env::var_os("PROGRAMDATA");
let original_local = std::env::var_os("LOCALAPPDATA");
unsafe {
std::env::set_var("PROGRAMDATA", r"C:\osdns-test-programdata-not-real");
std::env::set_var("LOCALAPPDATA", r"C:\osdns-test-localappdata-not-real");
}
let resolved = global_lock_root();
unsafe {
match original_programdata {
Some(value) => std::env::set_var("PROGRAMDATA", value),
None => std::env::remove_var("PROGRAMDATA"),
}
match original_local {
Some(value) => std::env::set_var("LOCALAPPDATA", value),
None => std::env::remove_var("LOCALAPPDATA"),
}
}
let path = resolved.expect("machine ProgramData must be resolvable");
assert!(
path.ends_with("osdns"),
"lock root should be under ProgramData\\osdns: {}",
path.display()
);
let text = path.to_string_lossy();
assert!(
!text.contains("osdns-test-programdata-not-real"),
"lock root followed PROGRAMDATA: {text}"
);
assert!(
!text.contains("osdns-test-localappdata-not-real"),
"lock root followed LOCALAPPDATA: {text}"
);
}
}