#[path = "diagnostics/disk_guard.rs"]
mod disk_guard;
#[path = "diagnostics/writer_contention.rs"]
mod writer_contention;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use khive_storage::error::StorageError;
use khive_storage::types::StorageResult;
use khive_storage::StorageCapability;
use rusqlite::Connection;
use serde::Serialize;
use crate::checkpoint;
use crate::pool::{ConnectionPool, WalCeilingSource};
pub use disk_guard::DiskGuardDiagnostics;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub struct CheckpointProbe {
pub busy: i64,
pub log_frames: i64,
pub checkpointed_frames: i64,
}
impl CheckpointProbe {
pub fn backfill_gap_frames(&self) -> i64 {
if self.log_frames < 0 || self.checkpointed_frames < 0 {
return 0;
}
self.log_frames
.saturating_sub(self.checkpointed_frames)
.max(0)
}
}
pub fn checkpoint_probe(conn: &Connection) -> rusqlite::Result<CheckpointProbe> {
conn.query_row("PRAGMA wal_checkpoint(PASSIVE)", [], |row| {
Ok(CheckpointProbe {
busy: row.get(0)?,
log_frames: row.get(1)?,
checkpointed_frames: row.get(2)?,
})
})
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub struct CheckpointCounters {
pub last_observed_wal_pages: Option<u64>,
pub truncate_attempts: u64,
pub truncate_consecutive_failures: u64,
pub checkpoint_skipped_ticks: u64,
pub checkpoint_consecutive_skips: u64,
pub checkpoint_last_skip_wal_pages: Option<u64>,
pub checkpoint_pressure_elevated_ticks: u64,
pub checkpoint_pressure_episodes_started: u64,
pub checkpoint_pressure_episodes_recovered: u64,
pub checkpoint_lifecycle_append_attempts: u64,
pub checkpoint_lifecycle_append_failures: u64,
pub checkpoint_lifecycle_enqueue_drops: u64,
pub read_tx_max_age_evictions: u64,
}
pub fn checkpoint_counters() -> CheckpointCounters {
CheckpointCounters {
last_observed_wal_pages: checkpoint::last_observed_wal_pages(),
truncate_attempts: checkpoint::truncate_attempts(),
truncate_consecutive_failures: checkpoint::truncate_consecutive_failures(),
checkpoint_skipped_ticks: checkpoint::checkpoint_skipped_ticks(),
checkpoint_consecutive_skips: checkpoint::checkpoint_consecutive_skips(),
checkpoint_last_skip_wal_pages: checkpoint::checkpoint_last_skip_wal_pages(),
checkpoint_pressure_elevated_ticks: checkpoint::checkpoint_pressure_elevated_ticks(),
checkpoint_pressure_episodes_started: checkpoint::checkpoint_pressure_episodes_started(),
checkpoint_pressure_episodes_recovered: checkpoint::checkpoint_pressure_episodes_recovered(
),
checkpoint_lifecycle_append_attempts: checkpoint::checkpoint_lifecycle_append_attempts(),
checkpoint_lifecycle_append_failures: checkpoint::checkpoint_lifecycle_append_failures(),
checkpoint_lifecycle_enqueue_drops: checkpoint::checkpoint_lifecycle_enqueue_drops(),
read_tx_max_age_evictions: checkpoint::read_tx_max_age_evictions(),
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct BuildIdentity {
pub version: String,
pub build_hash: Option<String>,
}
impl BuildIdentity {
pub fn from_env(version: &str, build_hash: Option<&str>) -> Self {
Self {
version: version.to_string(),
build_hash: build_hash.map(str::to_string),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct ProcessIdentity {
pub pid: u32,
pub started_at: Option<i64>,
pub started_at_unavailable_reason: Option<String>,
pub pool_generation: u64,
}
impl ProcessIdentity {
pub fn current(pool: &ConnectionPool) -> Self {
let pid = std::process::id();
Self::from_start_time(
pid,
crate::walpin::process_start_time_secs(pid),
pool.main_pool_generation(),
)
}
fn from_start_time(pid: u32, started_at: Option<i64>, pool_generation: u64) -> Self {
Self {
pid,
started_at,
started_at_unavailable_reason: started_at.is_none().then(|| {
format!(
"OS process start time is unsupported or unavailable on {}",
std::env::consts::OS
)
}),
pool_generation,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct WalFileState {
pub wal_path: String,
pub wal_size_bytes: Option<u64>,
pub unavailable_reason: Option<String>,
}
pub fn wal_file_state(db_path: &Path) -> WalFileState {
let wal_path = wal_sidecar_path(db_path);
match std::fs::metadata(&wal_path) {
Ok(md) => WalFileState {
wal_path: wal_path.display().to_string(),
wal_size_bytes: Some(md.len()),
unavailable_reason: None,
},
Err(e) => WalFileState {
wal_path: wal_path.display().to_string(),
wal_size_bytes: None,
unavailable_reason: Some(e.to_string()),
},
}
}
fn wal_sidecar_path(db_path: &Path) -> PathBuf {
let mut s = db_path.as_os_str().to_os_string();
s.push("-wal");
PathBuf::from(s)
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct WalPinAttribution {
pub status: WalPinAttributionStatus,
pub status_reasons: Vec<String>,
pub census: WalPinCensus,
pub reporting_pid: u32,
pub reporting_process_is_holder: Option<bool>,
pub reporting_process_is_holder_unavailable_reason: Option<String>,
pub census_process_start_times: Vec<WalPinCensusProcessStart>,
pub start_time_resolution_secs: Option<u64>,
pub start_time_resolution_unavailable_reason: Option<String>,
pub available: bool,
pub unavailable_reason: Option<String>,
#[serde(skip_serializing)]
pub census_holder_pids: Vec<u32>,
#[serde(skip_serializing)]
pub census_uninspectable_pids: Vec<u32>,
#[serde(skip_serializing)]
pub census_truncated: bool,
#[serde(skip_serializing)]
pub census_is_complete: bool,
pub reporting: Vec<WalPinHolder>,
pub registered_silent_pids: Vec<u32>,
pub unknown_pids: Vec<u32>,
pub census_pids_without_attribution: Vec<u32>,
pub fully_attributed: bool,
pub sidecar_entries: Vec<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub sidecar_listing_truncated: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub sidecar_entries_cleanup_would_reap: Option<usize>,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct WalPinCensusProcessStart {
pub pid: u32,
pub process_start_time_secs: Option<i64>,
pub process_start_time_unavailable_reason: Option<String>,
}
fn census_process_start_times(holder_pids: &[u32]) -> Vec<WalPinCensusProcessStart> {
holder_pids
.iter()
.map(|pid| {
let process_start_time_secs = crate::walpin::process_start_time_secs(*pid);
let process_start_time_unavailable_reason =
process_start_time_secs.is_none().then(|| {
if crate::walpin::start_time_resolution_secs().is_none() {
"process start time is unavailable on this platform".to_string()
} else {
"the operating system did not report a start time for this process"
.to_string()
}
});
WalPinCensusProcessStart {
pid: *pid,
process_start_time_secs,
process_start_time_unavailable_reason,
}
})
.collect()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum WalPinAttributionStatus {
Complete,
Degraded,
Unavailable,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(tag = "status", rename_all = "snake_case")]
pub enum WalPinCensus {
Complete {
holder_pids: Vec<u32>,
},
Incomplete {
holder_pids: Vec<u32>,
uninspectable_pids: Vec<u32>,
truncated: bool,
reason: String,
},
Unavailable {
reason: String,
},
}
fn census_truncation_cause(budget_exhausted: bool) -> String {
if budget_exhausted {
"the OS process walk stopped at its wall-clock budget (see collection_cost.wal_pin_census_budget_ms)"
.to_string()
} else {
"the OS process walk was truncated".to_string()
}
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct WalPinHolder {
pub pid: u32,
pub process_role: String,
pub current_oldest_tx_age_secs: f64,
pub oldest_tx_label: Option<String>,
pub attribution_is_evidence_backed: bool,
}
impl WalPinAttribution {
fn unavailable(reason: impl Into<String>) -> Self {
let reason = reason.into();
let start_time_resolution_secs = crate::walpin::start_time_resolution_secs();
Self {
status: WalPinAttributionStatus::Unavailable,
status_reasons: vec![reason.clone()],
census: WalPinCensus::Unavailable {
reason: reason.clone(),
},
reporting_pid: crate::walpin::reporting_pid(),
reporting_process_is_holder: None,
reporting_process_is_holder_unavailable_reason: Some(
"the OS holder census supplied no holder evidence".to_string(),
),
census_process_start_times: Vec::new(),
start_time_resolution_secs,
start_time_resolution_unavailable_reason: start_time_resolution_secs
.is_none()
.then(|| "process start time is unavailable on this platform".to_string()),
available: false,
unavailable_reason: Some(reason),
census_holder_pids: Vec::new(),
census_uninspectable_pids: Vec::new(),
census_truncated: false,
census_is_complete: false,
reporting: Vec::new(),
registered_silent_pids: Vec::new(),
unknown_pids: Vec::new(),
census_pids_without_attribution: Vec::new(),
fully_attributed: false,
sidecar_entries: Vec::new(),
sidecar_listing_truncated: None,
sidecar_entries_cleanup_would_reap: None,
}
}
}
#[cfg(all(unix, test))]
fn wal_pin_attribution_from_census(census: crate::walpin::CensusResult) -> WalPinAttribution {
wal_pin_attribution_without_sidecar(
census,
"read-only sidecar enumeration did not run for this attribution snapshot".to_string(),
)
}
#[cfg(unix)]
fn wal_pin_attribution_without_sidecar(
census: crate::walpin::CensusResult,
sidecar_reason: String,
) -> WalPinAttribution {
let census_is_complete = census.is_complete();
let mut census_holder_pids: Vec<u32> = census.holders.iter().copied().collect();
census_holder_pids.sort_unstable();
let reporting_pid = crate::walpin::reporting_pid();
let reporting_process_is_holder = census_holder_pids
.contains(&reporting_pid)
.then_some(true)
.or_else(|| census_is_complete.then_some(false));
let reporting_process_is_holder_unavailable_reason =
reporting_process_is_holder.is_none().then(|| {
"the reporting process was absent from the incomplete OS holder census".to_string()
});
let census_process_start_times = census_process_start_times(&census_holder_pids);
let start_time_resolution_secs = crate::walpin::start_time_resolution_secs();
let mut census_uninspectable_pids = census.uninspectable_pids;
census_uninspectable_pids.sort_unstable();
census_uninspectable_pids.dedup();
let census_truncated = census.truncated;
let census_budget_exhausted = census.budget_exhausted;
let mut status_reasons = vec![sidecar_reason];
let census = if census_is_complete {
WalPinCensus::Complete {
holder_pids: census_holder_pids.clone(),
}
} else {
let mut causes = Vec::new();
if census_truncated {
causes.push(census_truncation_cause(census_budget_exhausted));
}
if !census_uninspectable_pids.is_empty() {
causes.push(format!(
"{} PID(s) could not be inspected",
census_uninspectable_pids.len()
));
}
let reason = format!(
"OS holder census is incomplete: {}; additional database holders cannot be ruled out",
causes.join("; ")
);
status_reasons.push(reason.clone());
WalPinCensus::Incomplete {
holder_pids: census_holder_pids.clone(),
uninspectable_pids: census_uninspectable_pids.clone(),
truncated: census_truncated,
reason,
}
};
WalPinAttribution {
status: WalPinAttributionStatus::Degraded,
unavailable_reason: Some(status_reasons.join("; ")),
status_reasons,
census,
reporting_pid,
reporting_process_is_holder,
reporting_process_is_holder_unavailable_reason,
census_process_start_times,
start_time_resolution_secs,
start_time_resolution_unavailable_reason: start_time_resolution_secs
.is_none()
.then(|| "process start time is unavailable on this platform".to_string()),
available: false,
census_holder_pids,
census_uninspectable_pids,
census_truncated,
census_is_complete,
reporting: Vec::new(),
registered_silent_pids: Vec::new(),
unknown_pids: Vec::new(),
census_pids_without_attribution: Vec::new(),
fully_attributed: false,
sidecar_entries: Vec::new(),
sidecar_listing_truncated: None,
sidecar_entries_cleanup_would_reap: None,
}
}
#[cfg(unix)]
fn wal_pin_attribution_from_evidence(
census: crate::walpin::CensusResult,
sidecar: crate::walpin::WalpinReport,
) -> WalPinAttribution {
use std::collections::BTreeSet;
let census_is_complete = census.is_complete();
let mut census_holder_pids: Vec<u32> = census.holders.iter().copied().collect();
census_holder_pids.sort_unstable();
let reporting_pid = crate::walpin::reporting_pid();
let reporting_process_is_holder = census_holder_pids
.contains(&reporting_pid)
.then_some(true)
.or_else(|| census_is_complete.then_some(false));
let reporting_process_is_holder_unavailable_reason =
reporting_process_is_holder.is_none().then(|| {
"the reporting process was absent from the incomplete OS holder census".to_string()
});
let census_process_start_times = census_process_start_times(&census_holder_pids);
let start_time_resolution_secs = crate::walpin::start_time_resolution_secs();
let mut census_uninspectable_pids = census.uninspectable_pids;
census_uninspectable_pids.sort_unstable();
census_uninspectable_pids.dedup();
let census_truncated = census.truncated;
let census_budget_exhausted = census.budget_exhausted;
let census_carrier = if census_is_complete {
WalPinCensus::Complete {
holder_pids: census_holder_pids.clone(),
}
} else {
let mut causes = Vec::new();
if census_truncated {
causes.push(census_truncation_cause(census_budget_exhausted));
}
if !census_uninspectable_pids.is_empty() {
causes.push(format!(
"{} PID(s) could not be inspected",
census_uninspectable_pids.len()
));
}
let reason = format!(
"OS holder census is incomplete: {}; additional database holders cannot be ruled out",
causes.join("; ")
);
WalPinCensus::Incomplete {
holder_pids: census_holder_pids.clone(),
uninspectable_pids: census_uninspectable_pids.clone(),
truncated: census_truncated,
reason,
}
};
let now_epoch_secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_secs() as i64)
.unwrap_or(0);
let sidecar_listing_truncated = sidecar.sidecar_listing_truncated;
let sidecar_entries_cleanup_would_reap = sidecar.cleanup_would_reap;
let mut reporting = Vec::new();
let mut registered_silent_pids = Vec::new();
let mut unknown_pids = Vec::new();
let mut sidecar_entries = Vec::new();
let mut sidecar_known_pids = BTreeSet::new();
for entry in sidecar.entries {
match entry {
crate::walpin::WalpinPidHealth::Reporting(heartbeat) => {
let current_oldest_tx_age_secs =
heartbeat.current_oldest_tx_age_secs(now_epoch_secs);
let attribution_is_evidence_backed = heartbeat.attribution_is_evidence_backed();
sidecar_known_pids.insert(heartbeat.pid);
reporting.push(WalPinHolder {
pid: heartbeat.pid,
process_role: heartbeat.process_role.clone(),
current_oldest_tx_age_secs,
oldest_tx_label: heartbeat.oldest_tx_label.clone(),
attribution_is_evidence_backed,
});
sidecar_entries.push((
heartbeat.pid,
0u8,
serde_json::json!({
"pid": heartbeat.pid,
"status": "reporting",
"process_role": heartbeat.process_role,
"current_oldest_tx_age_secs": current_oldest_tx_age_secs,
"oldest_tx_label": heartbeat.oldest_tx_label,
"attribution_is_evidence_backed": attribution_is_evidence_backed,
}),
));
}
crate::walpin::WalpinPidHealth::RegisteredSilent { pid } => {
sidecar_known_pids.insert(pid);
registered_silent_pids.push(pid);
sidecar_entries.push((
pid,
1u8,
serde_json::json!({"pid": pid, "status": "registered_silent"}),
));
}
crate::walpin::WalpinPidHealth::Unknown { pid, reason } => {
sidecar_known_pids.insert(pid);
unknown_pids.push(pid);
sidecar_entries.push((
pid,
2u8,
serde_json::json!({"pid": pid, "status": "unknown", "reason": reason}),
));
}
}
}
reporting.sort_by_key(|holder| holder.pid);
reporting.dedup_by_key(|holder| holder.pid);
registered_silent_pids.sort_unstable();
registered_silent_pids.dedup();
unknown_pids.sort_unstable();
unknown_pids.dedup();
sidecar_entries.sort_by_key(|(pid, status_rank, _)| (*pid, *status_rank));
let sidecar_entries = sidecar_entries
.into_iter()
.map(|(_, _, entry)| entry)
.collect();
let census_pids_without_attribution: Vec<u32> = census_holder_pids
.iter()
.copied()
.filter(|pid| !sidecar_known_pids.contains(pid))
.collect();
let mut status_reasons = Vec::new();
if let WalPinCensus::Incomplete { reason, .. } = &census_carrier {
status_reasons.push(reason.clone());
}
if sidecar_listing_truncated {
status_reasons.push(
"read-only sidecar enumeration reached its entry cap; additional entries may exist"
.to_string(),
);
}
if !unknown_pids.is_empty() {
status_reasons.push(format!(
"{} sidecar PID(s) could not be classified conclusively",
unknown_pids.len()
));
}
if !census_pids_without_attribution.is_empty() {
status_reasons.push(format!(
"{} OS-confirmed holder(s) have no sidecar attribution",
census_pids_without_attribution.len()
));
}
let fully_attributed = census_is_complete
&& !sidecar_listing_truncated
&& unknown_pids.is_empty()
&& census_pids_without_attribution.is_empty();
let status = if fully_attributed {
WalPinAttributionStatus::Complete
} else {
WalPinAttributionStatus::Degraded
};
let unavailable_reason = (!fully_attributed).then(|| status_reasons.join("; "));
WalPinAttribution {
status,
status_reasons,
census: census_carrier,
reporting_pid,
reporting_process_is_holder,
reporting_process_is_holder_unavailable_reason,
census_process_start_times,
start_time_resolution_secs,
start_time_resolution_unavailable_reason: start_time_resolution_secs
.is_none()
.then(|| "process start time is unavailable on this platform".to_string()),
available: fully_attributed,
unavailable_reason,
census_holder_pids,
census_uninspectable_pids,
census_truncated,
census_is_complete,
reporting,
registered_silent_pids,
unknown_pids,
census_pids_without_attribution,
fully_attributed,
sidecar_entries,
sidecar_listing_truncated: Some(sidecar_listing_truncated),
sidecar_entries_cleanup_would_reap: Some(sidecar_entries_cleanup_would_reap),
}
}
#[cfg(unix)]
pub fn wal_pin_attribution(db_path: &Path, sweep_interval: Duration) -> WalPinAttribution {
use crate::walpin;
let census = match walpin::census_holders(db_path) {
Ok(c) => c,
Err(e) => return WalPinAttribution::unavailable(format!("census_holders failed: {e}")),
};
if !walpin::sidecar_enabled(true) {
return wal_pin_attribution_without_sidecar(census, SIDECAR_DISABLED_REASON.to_string());
}
match walpin::inspect_live(&walpin::sidecar_dir_for(db_path), sweep_interval) {
Ok(sidecar) => wal_pin_attribution_from_evidence(census, sidecar),
Err(error) => wal_pin_attribution_without_sidecar(
census,
format!("read-only sidecar enumeration failed: {error}"),
),
}
}
#[cfg(unix)]
const SIDECAR_DISABLED_REASON: &str =
"walpin sidecar is explicitly disabled (KHIVE_WALPIN_SIDECAR); attribution has no sidecar \
evidence to reconcile against the OS holder census";
#[cfg(not(unix))]
pub fn wal_pin_attribution(_db_path: &Path, _sweep_interval: Duration) -> WalPinAttribution {
WalPinAttribution::unavailable("WAL-pin attribution requires a Unix platform")
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub struct ReaderContentionDiagnostics {
pub configured_reader_cap: usize,
pub configured_checkout_timeout_ms: u64,
pub configured_busy_timeout_ms: u64,
pub reader_admission_capacity: usize,
pub available_reader_admission_slots: usize,
pub reader_acquisitions: u64,
pub pooled_reader_checkouts: u64,
pub standalone_reader_opens: u64,
pub infrastructure_standalone_reader_opens: u64,
pub reader_checkout_timeouts: u64,
pub reader_busy_timeouts: u64,
pub active_pooled_reader_checkouts: u64,
pub peak_active_pooled_reader_checkouts: u64,
pub completed_pooled_reader_checkouts: u64,
pub max_completed_reader_hold_micros: u64,
pub max_completed_reader_hold_operation: Option<&'static str>,
pub reader_discards: u64,
pub reader_replacement_open_failures: u64,
}
impl ReaderContentionDiagnostics {
fn snapshot(pool: &ConnectionPool) -> Self {
let reader = pool.reader_acquisition_snapshot();
Self {
configured_reader_cap: pool.config().max_readers,
configured_checkout_timeout_ms: u64::try_from(
pool.config().checkout_timeout.as_millis(),
)
.unwrap_or(u64::MAX),
configured_busy_timeout_ms: u64::try_from(pool.config().busy_timeout.as_millis())
.unwrap_or(u64::MAX),
reader_admission_capacity: reader.reader_admission_capacity,
available_reader_admission_slots: reader.available_reader_admission_slots,
reader_acquisitions: reader.acquisitions,
pooled_reader_checkouts: reader.pooled_checkouts,
standalone_reader_opens: reader.standalone_opens,
infrastructure_standalone_reader_opens: reader.infrastructure_standalone_opens,
reader_checkout_timeouts: reader.checkout_timeouts,
reader_busy_timeouts: reader.busy_timeouts,
active_pooled_reader_checkouts: reader.active_pooled_checkouts,
peak_active_pooled_reader_checkouts: reader.peak_active_pooled_checkouts,
completed_pooled_reader_checkouts: reader.completed_pooled_checkouts,
max_completed_reader_hold_micros: reader.max_completed_hold_micros,
max_completed_reader_hold_operation: reader.max_completed_hold_operation,
reader_discards: reader.reader_discards,
reader_replacement_open_failures: reader.reader_replacement_open_failures,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct WriterContentionDiagnostics {
pub writer_acquisitions: u64,
pub pooled_writer_acquisitions: u64,
pub standalone_writer_acquisitions: u64,
pub writer_task_acquisitions: u64,
pub writer_acquisition_timeouts: u64,
pub writer_lease_timeouts: u64,
pub configured_guard_deadline_ms: Option<u64>,
pub configured_checkout_timeout_ms: u64,
pub effective_writer_wait_bound_ms: u64,
pub direct_writer_busy_refusals: u64,
pub writer_task_begin_busy: u64,
pub writer_task_begin_busy_absorbed: u64,
pub writer_task_begin_errors: u64,
pub writer_task_request_failures: u64,
pub writer_task_side_effects_unknown: u64,
pub audit_append_failures: Option<u64>,
pub audit_append_failures_unavailable_reason: Option<String>,
pub audit_obligation_append_failures: Option<u64>,
pub audit_obligation_append_failures_unavailable_reason: Option<String>,
pub audit_batch_flush_failures: Option<u64>,
pub audit_batch_flush_failures_unavailable_reason: Option<String>,
pub audit_degraded_rows: Option<u64>,
pub audit_degraded_rows_unavailable_reason: Option<String>,
pub audit_degraded: Option<bool>,
pub audit_degraded_unavailable_reason: Option<String>,
pub audit_admission_refused_obligations: Option<u64>,
pub audit_admission_refused_obligations_last_at_ms: Option<u64>,
pub audit_admission_refused_obligations_unavailable_reason: Option<String>,
pub audit_retry_exhausted_obligations: Option<u64>,
pub audit_retry_exhausted_obligations_last_at_ms: Option<u64>,
pub audit_retry_exhausted_obligations_unavailable_reason: Option<String>,
pub audit_admission_unresolved_obligations: Option<u64>,
pub audit_admission_unresolved_obligations_last_at_ms: Option<u64>,
pub audit_admission_unresolved_obligations_unavailable_reason: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RuntimeAuditBatchMetrics {
pub flush_failures: u64,
pub degraded_rows: u64,
pub degraded: bool,
pub admission_refused_obligations: u64,
pub admission_refused_obligations_last_at_ms: Option<u64>,
pub retry_exhausted_obligations: u64,
pub retry_exhausted_obligations_last_at_ms: Option<u64>,
pub admission_unresolved_obligations: u64,
pub admission_unresolved_obligations_last_at_ms: Option<u64>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub struct GraphEdgeIntegrity {
pub duplicate_edge_id_groups: i64,
pub graph_edges_rows: i64,
pub graph_edges_seq_rows: i64,
pub pre_v14_duplicate_edge_state_detected: bool,
pub live_entities_carrying_merged_into: i64,
}
fn graph_edge_integrity(conn: &Connection) -> rusqlite::Result<GraphEdgeIntegrity> {
conn.query_row(
"SELECT
(SELECT COUNT(*) FROM (
SELECT id FROM graph_edges GROUP BY id HAVING COUNT(*) > 1
)),
(SELECT COUNT(*) FROM graph_edges),
(SELECT COUNT(*) FROM graph_edges_seq),
(SELECT COUNT(*) FROM entities
WHERE deleted_at IS NULL AND merged_into IS NOT NULL)",
[],
|row| {
let duplicate_edge_id_groups = row.get(0)?;
Ok(GraphEdgeIntegrity {
duplicate_edge_id_groups,
graph_edges_rows: row.get(1)?,
graph_edges_seq_rows: row.get(2)?,
pre_v14_duplicate_edge_state_detected: duplicate_edge_id_groups > 0,
live_entities_carrying_merged_into: row.get(3)?,
})
},
)
}
const MAX_DATABASE_SIZE_OBJECTS: usize = 4_096;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum DatabaseObjectKind {
Table,
Index,
Internal,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum DatabaseStorageClass {
RowTable,
Index,
FullText,
Vector,
MixedRowAndEmbedding,
Internal,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct DatabaseObjectSize {
pub name: String,
pub owner_table: Option<String>,
pub object_kind: DatabaseObjectKind,
pub storage_class: DatabaseStorageClass,
pub pages: u64,
pub bytes: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct DatabaseSizeComposition {
pub page_size_bytes: u64,
pub page_count: u64,
pub freelist_pages: u64,
pub database_bytes: u64,
pub freelist_bytes: u64,
pub accounted_bytes: u64,
pub unaccounted_bytes: u64,
pub row_table_bytes: u64,
pub index_bytes: u64,
pub full_text_bytes: u64,
pub vector_bytes: u64,
pub mixed_embedding_bytes: u64,
pub internal_bytes: u64,
pub objects: Vec<DatabaseObjectSize>,
pub objects_truncated: bool,
pub objects_omitted: usize,
}
fn nonnegative_sqlite_integer(column: usize, value: i64) -> rusqlite::Result<u64> {
u64::try_from(value).map_err(|_| rusqlite::Error::IntegralValueOutOfRange(column, value))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub struct DatabasePageAccounting {
pub page_size_bytes: u64,
pub page_count: u64,
pub freelist_pages: u64,
}
fn database_page_accounting(conn: &Connection) -> rusqlite::Result<DatabasePageAccounting> {
let read = |pragma: &str| -> rusqlite::Result<u64> {
nonnegative_sqlite_integer(0, conn.query_row(pragma, [], |row| row.get::<_, i64>(0))?)
};
Ok(DatabasePageAccounting {
page_size_bytes: read("PRAGMA page_size")?,
page_count: read("PRAGMA page_count")?,
freelist_pages: read("PRAGMA freelist_count")?,
})
}
fn declares_embedding_blob(sql: Option<&str>) -> bool {
let Some(sql) = sql else {
return false;
};
let tokens: Vec<_> = sql
.split(|character: char| !(character.is_ascii_alphanumeric() || character == '_'))
.filter(|token| !token.is_empty())
.collect();
tokens.windows(2).any(|pair| {
pair[0].eq_ignore_ascii_case("embedding") && pair[1].eq_ignore_ascii_case("blob")
})
}
fn classify_database_object(
name: &str,
sqlite_type: &str,
sql: Option<&str>,
) -> (DatabaseObjectKind, DatabaseStorageClass) {
let object_kind = match sqlite_type {
"table" => DatabaseObjectKind::Table,
"index" => DatabaseObjectKind::Index,
_ => DatabaseObjectKind::Internal,
};
let lower_name = name.to_ascii_lowercase();
let lower_sql = sql.unwrap_or_default().to_ascii_lowercase();
let storage_class = if lower_name.starts_with("fts_") || lower_sql.contains("using fts5") {
DatabaseStorageClass::FullText
} else if lower_name.starts_with("vec_")
|| lower_name == "_embedding_models"
|| lower_sql.contains("using vec0")
{
DatabaseStorageClass::Vector
} else if declares_embedding_blob(sql) {
DatabaseStorageClass::MixedRowAndEmbedding
} else if object_kind == DatabaseObjectKind::Index {
DatabaseStorageClass::Index
} else if name.starts_with("sqlite_") || object_kind == DatabaseObjectKind::Internal {
DatabaseStorageClass::Internal
} else {
DatabaseStorageClass::RowTable
};
(object_kind, storage_class)
}
fn database_size_composition(conn: &Connection) -> rusqlite::Result<DatabaseSizeComposition> {
let DatabasePageAccounting {
page_size_bytes: page_size,
page_count,
freelist_pages,
} = database_page_accounting(conn)?;
let mut statement = conn.prepare(
"SELECT d.name, COALESCE(s.type, 'internal'), s.tbl_name, s.sql, d.pageno, d.pgsize
FROM dbstat AS d
LEFT JOIN sqlite_schema AS s ON s.name = d.name
WHERE d.aggregate = TRUE
ORDER BY d.name",
)?;
let mut rows = statement.query([])?;
let mut objects = Vec::new();
let mut objects_omitted = 0usize;
let mut accounted_bytes = 0u64;
let mut row_table_bytes = 0u64;
let mut index_bytes = 0u64;
let mut full_text_bytes = 0u64;
let mut vector_bytes = 0u64;
let mut mixed_embedding_bytes = 0u64;
let mut internal_bytes = 0u64;
while let Some(row) = rows.next()? {
let name: String = row.get(0)?;
let sqlite_type: String = row.get(1)?;
let owner_table: Option<String> = row.get(2)?;
let sql: Option<String> = row.get(3)?;
let pages = nonnegative_sqlite_integer(4, row.get(4)?)?;
let bytes = nonnegative_sqlite_integer(5, row.get(5)?)?;
let (object_kind, storage_class) =
classify_database_object(&name, &sqlite_type, sql.as_deref());
accounted_bytes = accounted_bytes.saturating_add(bytes);
let class_total = match storage_class {
DatabaseStorageClass::RowTable => &mut row_table_bytes,
DatabaseStorageClass::Index => &mut index_bytes,
DatabaseStorageClass::FullText => &mut full_text_bytes,
DatabaseStorageClass::Vector => &mut vector_bytes,
DatabaseStorageClass::MixedRowAndEmbedding => &mut mixed_embedding_bytes,
DatabaseStorageClass::Internal => &mut internal_bytes,
};
*class_total = class_total.saturating_add(bytes);
if objects.len() < MAX_DATABASE_SIZE_OBJECTS {
objects.push(DatabaseObjectSize {
name,
owner_table,
object_kind,
storage_class,
pages,
bytes,
});
} else {
objects_omitted = objects_omitted.saturating_add(1);
}
}
let database_bytes = page_count.saturating_mul(page_size);
let freelist_bytes = freelist_pages.saturating_mul(page_size);
let unaccounted_bytes = database_bytes
.saturating_sub(freelist_bytes)
.saturating_sub(accounted_bytes);
Ok(DatabaseSizeComposition {
page_size_bytes: page_size,
page_count,
freelist_pages,
database_bytes,
freelist_bytes,
accounted_bytes,
unaccounted_bytes,
row_table_bytes,
index_bytes,
full_text_bytes,
vector_bytes,
mixed_embedding_bytes,
internal_bytes,
objects,
objects_truncated: objects_omitted > 0,
objects_omitted,
})
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub struct CollectionCost {
pub total_ms: u64,
pub sqlite_ms: u64,
pub wal_file_stat_ms: u64,
pub wal_pin_census_ms: u64,
pub wal_pin_sidecar_ms: u64,
pub wal_pin_census_budget_ms: Option<u64>,
pub wal_pin_census_budget_exhausted: bool,
}
impl CollectionCost {
fn in_memory(total_ms: u64) -> Self {
Self {
total_ms,
sqlite_ms: 0,
wal_file_stat_ms: 0,
wal_pin_census_ms: 0,
wal_pin_sidecar_ms: 0,
wal_pin_census_budget_ms: None,
wal_pin_census_budget_exhausted: false,
}
}
}
const DEFAULT_CENSUS_BUDGET: Duration = Duration::from_millis(2000);
const CENSUS_BUDGET_ENV: &str = "KHIVE_WALPIN_CENSUS_BUDGET_MS";
fn request_census_budget() -> Option<Duration> {
match std::env::var(CENSUS_BUDGET_ENV) {
Ok(raw) => match raw.trim().parse::<u64>() {
Ok(0) => None,
Ok(ms) => Some(Duration::from_millis(ms)),
Err(_) => Some(DEFAULT_CENSUS_BUDGET),
},
Err(_) => Some(DEFAULT_CENSUS_BUDGET),
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub struct WalCeilingDiagnostics {
pub configured_bytes: u64,
pub effective_bytes: u64,
pub source: WalCeilingSource,
pub enabled: bool,
pub status: &'static str,
}
impl WalCeilingDiagnostics {
fn from_pool(pool: &ConnectionPool) -> Self {
let policy = pool.config().wal_ceiling;
let read_only = pool.config().read_only;
let effective_bytes = policy.effective_bytes(read_only);
Self {
configured_bytes: policy.bytes,
effective_bytes,
source: policy.source,
enabled: effective_bytes > 0,
status: if effective_bytes > 0 {
"enforced"
} else if read_only && policy.bytes > 0 {
"read_only_not_enforced"
} else {
"disabled"
},
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct DbDiagnostics {
pub build: BuildIdentity,
pub process: ProcessIdentity,
pub note_search_ann_route_total: u64,
pub note_search_fallback_route_total: u64,
pub search_mechanism: crate::pool::SearchMechanismSnapshot,
pub db_path: Option<String>,
pub wal_ceiling: WalCeilingDiagnostics,
pub disk_guard: DiskGuardDiagnostics,
pub wal_file: Option<WalFileState>,
pub checkpoint_counters: CheckpointCounters,
pub checkpoint_probe: Option<CheckpointProbe>,
pub checkpoint_probe_error: Option<String>,
#[serde(flatten)]
pub checkpoint_pin: CheckpointPinDiagnostics,
pub reader_contention: ReaderContentionDiagnostics,
pub writer_contention: WriterContentionDiagnostics,
pub size_composition: Option<DatabaseSizeComposition>,
pub size_composition_error: Option<String>,
pub page_accounting: Option<DatabasePageAccounting>,
pub page_accounting_error: Option<String>,
pub graph_edge_integrity: Option<GraphEdgeIntegrity>,
pub graph_edge_integrity_error: Option<String>,
pub fts_segments: Option<crate::FtsSegmentDiagnostics>,
pub fts_segments_error: Option<String>,
pub fts_maintenance: crate::FtsMaintenanceCounters,
pub wal_pin: WalPinAttribution,
pub collection_cost: CollectionCost,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct CheckpointPinDiagnostics {
pub backfill_ceiling: Option<i64>,
pub backfill_ceiling_unavailable_reason: Option<String>,
pub oldest_pinned_frame: Option<i64>,
pub oldest_pinned_frame_unavailable_reason: Option<String>,
pub oldest_pinned_frame_run: Option<crate::checkpoint::CheckpointRun>,
pub oldest_pinned_frame_run_unavailable_reason: Option<String>,
pub pin_depth: Option<i64>,
pub pin_depth_unavailable_reason: Option<String>,
}
fn checkpoint_pin_diagnostics(
pool: &ConnectionPool,
probe: Option<&CheckpointProbe>,
probe_error: Option<&str>,
) -> CheckpointPinDiagnostics {
let (run_status, run_age) = checkpoint::checkpoint_run_snapshot(pool);
checkpoint_pin_diagnostics_for_run(probe, probe_error, run_status, run_age)
}
fn checkpoint_pin_diagnostics_for_run(
probe: Option<&CheckpointProbe>,
probe_error: Option<&str>,
run_status: checkpoint::CheckpointRunStatus,
run_age: Option<Duration>,
) -> CheckpointPinDiagnostics {
let (backfill_ceiling, backfill_reason) = match probe {
None => (
None,
Some(
probe_error
.unwrap_or("checkpoint probe did not return a row")
.to_string(),
),
),
Some(probe) if probe.busy != 0 => (
None,
Some(format!("PASSIVE checkpoint returned busy={}", probe.busy)),
),
Some(probe) if probe.log_frames < 0 || probe.checkpointed_frames < 0 => (
None,
Some("PASSIVE checkpoint returned a negative frame count".to_string()),
),
Some(probe) if probe.log_frames <= probe.checkpointed_frames => (
None,
Some("PASSIVE checkpoint found no frames beyond the backfill ceiling".to_string()),
),
Some(probe) => (Some(probe.checkpointed_frames), None),
};
let (oldest_pinned_frame, oldest_reason, oldest_pinned_frame_run, run_reason) =
match backfill_ceiling {
None => {
let reason = backfill_reason
.clone()
.unwrap_or_else(|| "backfill ceiling is unavailable".to_string());
(None, Some(reason.clone()), None, Some(reason))
}
Some(ceiling) => match run_status {
checkpoint::CheckpointRunStatus::NoTask => {
let reason = "no checkpoint task in this process".to_string();
(None, Some(reason.clone()), None, Some(reason))
}
checkpoint::CheckpointRunStatus::NoObservation => {
let reason = "no checkpoint run has been observed for this backend".to_string();
(None, Some(reason.clone()), None, Some(reason))
}
checkpoint::CheckpointRunStatus::Observed(run) if run.frame != ceiling => {
let reason = "checkpoint run frame does not match the current backfill ceiling";
(
None,
Some(reason.to_string()),
None,
Some(reason.to_string()),
)
}
checkpoint::CheckpointRunStatus::Observed(_)
if run_age.is_none_or(|age| age < Duration::from_secs(1)) =>
{
let reason = "checkpoint run has been observed for less than one second";
(
None,
Some(reason.to_string()),
None,
Some(reason.to_string()),
)
}
checkpoint::CheckpointRunStatus::Observed(run) => {
(Some(ceiling), None, Some(run), None)
}
},
};
let pin_depth = oldest_pinned_frame.map(|frame| {
probe
.expect("a reported pin always has a probe row")
.log_frames
.saturating_sub(frame)
.max(0)
});
let pin_depth_reason = if pin_depth.is_none() {
oldest_reason.clone()
} else {
None
};
CheckpointPinDiagnostics {
backfill_ceiling,
backfill_ceiling_unavailable_reason: backfill_reason,
oldest_pinned_frame,
oldest_pinned_frame_unavailable_reason: oldest_reason,
oldest_pinned_frame_run,
oldest_pinned_frame_run_unavailable_reason: run_reason,
pin_depth,
pin_depth_unavailable_reason: pin_depth_reason,
}
}
pub fn collect(
pool: &ConnectionPool,
build: BuildIdentity,
sweep_interval: Duration,
) -> DbDiagnostics {
collect_inner(pool, build, sweep_interval, None, None)
}
pub fn collect_with_audit_append_failures(
pool: &ConnectionPool,
build: BuildIdentity,
sweep_interval: Duration,
audit_append_failures: u64,
) -> DbDiagnostics {
collect_inner(
pool,
build,
sweep_interval,
Some(audit_append_failures),
None,
)
}
pub async fn collect_with_audit_append_failures_interruptibly(
pool: Arc<ConnectionPool>,
build: BuildIdentity,
sweep_interval: Duration,
audit_append_failures: u64,
) -> StorageResult<DbDiagnostics> {
collect_with_runtime_audit_metrics_interruptibly(
pool,
build,
sweep_interval,
audit_append_failures,
None,
)
.await
}
pub async fn collect_with_runtime_audit_metrics_interruptibly(
pool: Arc<ConnectionPool>,
build: BuildIdentity,
sweep_interval: Duration,
audit_append_failures: u64,
runtime_audit_batch_metrics: Option<RuntimeAuditBatchMetrics>,
) -> StorageResult<DbDiagnostics> {
let process = ProcessIdentity::current(&pool);
collect_with_runtime_audit_metrics_for_process_interruptibly(
pool,
build,
process,
sweep_interval,
audit_append_failures,
runtime_audit_batch_metrics,
true,
)
.await
}
pub async fn collect_with_runtime_audit_metrics_for_process_interruptibly(
pool: Arc<ConnectionPool>,
build: BuildIdentity,
process: ProcessIdentity,
sweep_interval: Duration,
audit_append_failures: u64,
runtime_audit_batch_metrics: Option<RuntimeAuditBatchMetrics>,
collect_size_composition: bool,
) -> StorageResult<DbDiagnostics> {
crate::ensure_request_read_active("db_diagnostics")?;
let started = Instant::now();
let counters = checkpoint_counters();
let reader_contention = ReaderContentionDiagnostics::snapshot(&pool);
let search_mechanism = pool.search_mechanism_snapshot();
let writer_contention = WriterContentionDiagnostics::snapshot(
&pool,
Some(audit_append_failures),
runtime_audit_batch_metrics,
);
let Some(path) = pool.config().path.clone() else {
crate::ensure_request_read_active("db_diagnostics")?;
return Ok(DbDiagnostics {
build,
process,
note_search_ann_route_total: 0,
note_search_fallback_route_total: 0,
search_mechanism,
db_path: None,
wal_ceiling: WalCeilingDiagnostics::from_pool(&pool),
disk_guard: DiskGuardDiagnostics::snapshot(&pool),
wal_file: None,
checkpoint_counters: counters,
checkpoint_probe: None,
checkpoint_probe_error: Some(
"in-memory database: no WAL file and no checkpoint to probe".to_string(),
),
checkpoint_pin: checkpoint_pin_diagnostics(
&pool,
None,
Some("in-memory database: no WAL file and no checkpoint to probe"),
),
reader_contention,
writer_contention,
size_composition: None,
size_composition_error: Some(
"in-memory database: no file-backed page composition to inspect".to_string(),
),
page_accounting: None,
page_accounting_error: Some(
"in-memory database: no file-backed pages to count".to_string(),
),
graph_edge_integrity: None,
graph_edge_integrity_error: Some(
"in-memory database: no durable graph-edge ledger to inspect".to_string(),
),
fts_segments: None,
fts_segments_error: Some(
"in-memory database: no durable FTS5 indexes to inspect".to_string(),
),
fts_maintenance: crate::fts_maintenance_counters(),
wal_pin: WalPinAttribution::unavailable(
"in-memory database: no file for the OS holder census",
),
collection_cost: CollectionCost::in_memory(elapsed_ms(started)),
});
};
let inspection_pool = Arc::clone(&pool);
let sqlite_started = Instant::now();
let inspection = crate::read_cancellation::run_interruptible_read(
StorageCapability::Sql,
"db_diagnostics.sqlite",
move |scope| inspect_pool_interruptibly(&inspection_pool, scope, collect_size_composition),
)
.await?;
let sqlite_ms = elapsed_ms(sqlite_started);
crate::ensure_request_read_active("db_diagnostics")?;
let canonical = operational_db_path(&pool, &path);
let budget = request_census_budget();
let (wal_file, wal_pin, file_state_cost) =
inspect_file_state_interruptibly(canonical, sweep_interval, budget).await?;
crate::ensure_request_read_active("db_diagnostics")?;
Ok(DbDiagnostics {
build,
process,
note_search_ann_route_total: 0,
note_search_fallback_route_total: 0,
search_mechanism,
db_path: Some(path.display().to_string()),
wal_ceiling: WalCeilingDiagnostics::from_pool(&pool),
disk_guard: DiskGuardDiagnostics::snapshot(&pool),
wal_file: Some(wal_file),
checkpoint_counters: counters,
checkpoint_probe: inspection.checkpoint_probe,
checkpoint_probe_error: inspection.checkpoint_probe_error,
checkpoint_pin: inspection.checkpoint_pin,
reader_contention,
writer_contention,
size_composition: inspection.size_composition,
size_composition_error: inspection.size_composition_error,
page_accounting: inspection.page_accounting,
page_accounting_error: inspection.page_accounting_error,
graph_edge_integrity: inspection.graph_edge_integrity,
graph_edge_integrity_error: inspection.graph_edge_integrity_error,
fts_segments: inspection.fts_segments,
fts_segments_error: inspection.fts_segments_error,
fts_maintenance: crate::fts_maintenance_counters(),
wal_pin,
collection_cost: CollectionCost {
total_ms: elapsed_ms(started),
sqlite_ms,
wal_file_stat_ms: file_state_cost.wal_file_stat_ms,
wal_pin_census_ms: file_state_cost.census_ms,
wal_pin_sidecar_ms: file_state_cost.sidecar_ms,
wal_pin_census_budget_ms: budget.map(|b| b.as_millis() as u64),
wal_pin_census_budget_exhausted: file_state_cost.census_budget_exhausted,
},
})
}
fn operational_db_path(pool: &ConnectionPool, configured: &Path) -> PathBuf {
pool.canonical_path()
.map(Path::to_path_buf)
.unwrap_or_else(|| configured.to_path_buf())
}
fn collect_inner(
pool: &ConnectionPool,
build: BuildIdentity,
sweep_interval: Duration,
audit_append_failures: Option<u64>,
runtime_audit_batch_metrics: Option<RuntimeAuditBatchMetrics>,
) -> DbDiagnostics {
let started = Instant::now();
let process = ProcessIdentity::current(pool);
let counters = checkpoint_counters();
let reader_contention = ReaderContentionDiagnostics::snapshot(pool);
let search_mechanism = pool.search_mechanism_snapshot();
let writer_contention = WriterContentionDiagnostics::snapshot(
pool,
audit_append_failures,
runtime_audit_batch_metrics,
);
let Some(path) = pool.config().path.clone() else {
return DbDiagnostics {
build,
process,
note_search_ann_route_total: 0,
note_search_fallback_route_total: 0,
search_mechanism,
db_path: None,
wal_ceiling: WalCeilingDiagnostics::from_pool(pool),
disk_guard: DiskGuardDiagnostics::snapshot(pool),
wal_file: None,
checkpoint_counters: counters,
checkpoint_probe: None,
checkpoint_probe_error: Some(
"in-memory database: no WAL file and no checkpoint to probe".to_string(),
),
checkpoint_pin: checkpoint_pin_diagnostics(
pool,
None,
Some("in-memory database: no WAL file and no checkpoint to probe"),
),
reader_contention,
writer_contention,
size_composition: None,
size_composition_error: Some(
"in-memory database: no file-backed page composition to inspect".to_string(),
),
page_accounting: None,
page_accounting_error: Some(
"in-memory database: no file-backed pages to count".to_string(),
),
graph_edge_integrity: None,
graph_edge_integrity_error: Some(
"in-memory database: no durable graph-edge ledger to inspect".to_string(),
),
fts_segments: None,
fts_segments_error: Some(
"in-memory database: no durable FTS5 indexes to inspect".to_string(),
),
fts_maintenance: crate::fts_maintenance_counters(),
wal_pin: WalPinAttribution::unavailable(
"in-memory database: no file for the OS holder census",
),
collection_cost: CollectionCost::in_memory(elapsed_ms(started)),
};
};
let sqlite_started = Instant::now();
let inspection = inspect_pool(pool);
let sqlite_ms = elapsed_ms(sqlite_started);
let canonical = operational_db_path(pool, &path);
let wal_file_started = Instant::now();
let wal_file = wal_file_state(&canonical);
let wal_file_stat_ms = elapsed_ms(wal_file_started);
let census_started = Instant::now();
let wal_pin = wal_pin_attribution(&canonical, sweep_interval);
let wal_pin_ms = elapsed_ms(census_started);
DbDiagnostics {
build,
process,
note_search_ann_route_total: 0,
note_search_fallback_route_total: 0,
search_mechanism,
db_path: Some(path.display().to_string()),
wal_ceiling: WalCeilingDiagnostics::from_pool(pool),
disk_guard: DiskGuardDiagnostics::snapshot(pool),
wal_file: Some(wal_file),
checkpoint_counters: counters,
checkpoint_probe: inspection.checkpoint_probe,
checkpoint_probe_error: inspection.checkpoint_probe_error,
checkpoint_pin: inspection.checkpoint_pin,
reader_contention,
writer_contention,
size_composition: inspection.size_composition,
size_composition_error: inspection.size_composition_error,
page_accounting: inspection.page_accounting,
page_accounting_error: inspection.page_accounting_error,
graph_edge_integrity: inspection.graph_edge_integrity,
graph_edge_integrity_error: inspection.graph_edge_integrity_error,
fts_segments: inspection.fts_segments,
fts_segments_error: inspection.fts_segments_error,
fts_maintenance: crate::fts_maintenance_counters(),
wal_pin,
collection_cost: CollectionCost {
total_ms: elapsed_ms(started),
sqlite_ms,
wal_file_stat_ms,
wal_pin_census_ms: wal_pin_ms,
wal_pin_sidecar_ms: 0,
wal_pin_census_budget_ms: None,
wal_pin_census_budget_exhausted: false,
},
}
}
struct PoolInspection {
checkpoint_probe: Option<CheckpointProbe>,
checkpoint_probe_error: Option<String>,
checkpoint_pin: CheckpointPinDiagnostics,
size_composition: Option<DatabaseSizeComposition>,
size_composition_error: Option<String>,
page_accounting: Option<DatabasePageAccounting>,
page_accounting_error: Option<String>,
graph_edge_integrity: Option<GraphEdgeIntegrity>,
graph_edge_integrity_error: Option<String>,
fts_segments: Option<crate::FtsSegmentDiagnostics>,
fts_segments_error: Option<String>,
}
fn split_fts_segments_result(
result: Result<crate::FtsSegmentDiagnostics, String>,
) -> (Option<crate::FtsSegmentDiagnostics>, Option<String>) {
match result {
Ok(segments) => (Some(segments), None),
Err(error) => (None, Some(error)),
}
}
fn split_page_accounting_result(
result: rusqlite::Result<DatabasePageAccounting>,
) -> (Option<DatabasePageAccounting>, Option<String>) {
match result {
Ok(pages) => (Some(pages), None),
Err(error) => (None, Some(format!("page accounting query failed: {error}"))),
}
}
pub const SIZE_COMPOSITION_NOT_REQUESTED: &str =
"not collected: size_composition=false skips the dbstat walk, which reads every page";
fn record_diagnostic_checkpoint_probe(
pool: &ConnectionPool,
result: &rusqlite::Result<CheckpointProbe>,
) -> (checkpoint::CheckpointRunStatus, Option<Duration>) {
checkpoint::record_checkpoint_run_result(
pool,
result
.as_ref()
.ok()
.map(|probe| (probe.busy, probe.log_frames, probe.checkpointed_frames)),
);
checkpoint::checkpoint_run_snapshot(pool)
}
fn inspect_pool_interruptibly(
pool: &ConnectionPool,
scope: &crate::read_cancellation::InterruptibleReadScope,
collect_size_composition: bool,
) -> StorageResult<PoolInspection> {
scope.ensure_active()?;
let conn = match pool.open_standalone_writer_untracked() {
Ok(conn) => conn,
Err(e) => {
scope.ensure_active()?;
let reason = format!("guarded standalone open refused: {e}");
return Ok(PoolInspection {
checkpoint_probe: None,
checkpoint_probe_error: Some(reason.clone()),
checkpoint_pin: checkpoint_pin_diagnostics(pool, None, Some(&reason)),
size_composition: None,
size_composition_error: Some(reason.clone()),
page_accounting: None,
page_accounting_error: Some(reason.clone()),
graph_edge_integrity: None,
graph_edge_integrity_error: Some(reason.clone()),
fts_segments: None,
fts_segments_error: Some(reason),
});
}
};
scope.ensure_active()?;
let probe_result = checkpoint_probe(&conn);
let (run_status, run_age) = record_diagnostic_checkpoint_probe(pool, &probe_result);
let (checkpoint_probe, checkpoint_probe_error) = match probe_result {
Ok(probe) => (Some(probe), None),
Err(e) => (
None,
Some(format!("PRAGMA wal_checkpoint(PASSIVE) failed: {e}")),
),
};
let checkpoint_pin = checkpoint_pin_diagnostics_for_run(
checkpoint_probe.as_ref(),
checkpoint_probe_error.as_deref(),
run_status,
run_age,
);
#[cfg(test)]
if TEST_PAUSE_AFTER_PASSIVE.load(Ordering::SeqCst) {
TEST_REACHED_AFTER_PASSIVE.store(true, Ordering::SeqCst);
while TEST_PAUSE_AFTER_PASSIVE.load(Ordering::SeqCst) && !scope.should_stop() {
std::thread::yield_now();
}
}
scope.ensure_active()?;
let (integrity, page_accounting, size_composition, fts_segments) = scope.run(&conn, || {
Ok((
graph_edge_integrity(&conn),
database_page_accounting(&conn),
collect_size_composition.then(|| database_size_composition(&conn)),
crate::fts_maintenance::inspect_fts_segments(&conn),
))
})?;
let (graph_edge_integrity, graph_edge_integrity_error) = match integrity {
Ok(integrity) => (Some(integrity), None),
Err(e) => (
None,
Some(format!("graph-edge integrity query failed: {e}")),
),
};
let (fts_segments, fts_segments_error) = split_fts_segments_result(fts_segments);
let (page_accounting, page_accounting_error) = split_page_accounting_result(page_accounting);
let (size_composition, size_composition_error) = match size_composition {
Some(Ok(composition)) => (Some(composition), None),
Some(Err(error)) => (
None,
Some(format!("database size composition query failed: {error}")),
),
None => (None, Some(SIZE_COMPOSITION_NOT_REQUESTED.to_string())),
};
Ok(PoolInspection {
checkpoint_probe,
checkpoint_probe_error,
checkpoint_pin,
size_composition,
size_composition_error,
page_accounting,
page_accounting_error,
graph_edge_integrity,
graph_edge_integrity_error,
fts_segments,
fts_segments_error,
})
}
#[cfg(test)]
static TEST_PAUSE_AFTER_PASSIVE: AtomicBool = AtomicBool::new(false);
#[cfg(test)]
static TEST_REACHED_AFTER_PASSIVE: AtomicBool = AtomicBool::new(false);
struct StopCensusOnDrop {
stopped: Arc<AtomicBool>,
armed: bool,
}
impl Drop for StopCensusOnDrop {
fn drop(&mut self) {
if self.armed {
self.stopped.store(true, Ordering::SeqCst);
}
}
}
#[derive(Debug, Clone, Copy, Default)]
struct FileStateCost {
wal_file_stat_ms: u64,
census_ms: u64,
sidecar_ms: u64,
census_budget_exhausted: bool,
}
fn elapsed_ms(since: Instant) -> u64 {
since.elapsed().as_millis() as u64
}
async fn inspect_file_state_interruptibly(
path: PathBuf,
sweep_interval: Duration,
census_budget: Option<Duration>,
) -> StorageResult<(WalFileState, WalPinAttribution, FileStateCost)> {
const OPERATION: &str = "db_diagnostics.wal_holder_census";
crate::ensure_request_read_active(OPERATION)?;
let stopped = Arc::new(AtomicBool::new(false));
let worker_stopped = Arc::clone(&stopped);
let mut stop_on_drop = StopCensusOnDrop {
stopped: Arc::clone(&stopped),
armed: true,
};
let mut worker = tokio::task::spawn_blocking(move || {
let mut cost = FileStateCost::default();
let wal_file_started = Instant::now();
let wal_file = wal_file_state(&path);
cost.wal_file_stat_ms = elapsed_ms(wal_file_started);
if worker_stopped.load(Ordering::SeqCst) {
return Err(std::io::Error::new(
std::io::ErrorKind::Interrupted,
"WAL holder census cancelled",
));
}
#[cfg(unix)]
let census_started = Instant::now();
#[cfg(unix)]
let census_result = match census_budget {
Some(budget) => crate::walpin::census_holders_until_within(
&path,
|| worker_stopped.load(Ordering::SeqCst),
budget,
),
None => {
crate::walpin::census_holders_until(&path, || worker_stopped.load(Ordering::SeqCst))
}
};
#[cfg(unix)]
{
cost.census_ms = elapsed_ms(census_started);
}
#[cfg(unix)]
let attribution = match census_result {
Ok(census) => {
cost.census_budget_exhausted = census.budget_exhausted;
if worker_stopped.load(Ordering::SeqCst) {
return Err(std::io::Error::new(
std::io::ErrorKind::Interrupted,
"WAL sidecar inspection cancelled",
));
}
if !crate::walpin::sidecar_enabled(true) {
wal_pin_attribution_without_sidecar(census, SIDECAR_DISABLED_REASON.to_string())
} else {
let sidecar_started = Instant::now();
let sidecar = crate::walpin::inspect_live(
&crate::walpin::sidecar_dir_for(&path),
sweep_interval,
);
cost.sidecar_ms = elapsed_ms(sidecar_started);
if worker_stopped.load(Ordering::SeqCst) {
return Err(std::io::Error::new(
std::io::ErrorKind::Interrupted,
"WAL sidecar inspection cancelled",
));
}
match sidecar {
Ok(sidecar) => wal_pin_attribution_from_evidence(census, sidecar),
Err(error) => wal_pin_attribution_without_sidecar(
census,
format!("read-only sidecar enumeration failed: {error}"),
),
}
}
}
Err(error) if error.kind() == std::io::ErrorKind::Interrupted => return Err(error),
Err(error) => WalPinAttribution::unavailable(format!("census_holders failed: {error}")),
};
#[cfg(not(unix))]
let attribution = {
let _ = census_budget;
let census_started = Instant::now();
let attribution = wal_pin_attribution(&path, sweep_interval);
cost.census_ms = elapsed_ms(census_started);
attribution
};
Ok((wal_file, attribution, cost))
});
tokio::select! {
joined = &mut worker => {
stop_on_drop.armed = false;
let result = joined
.map_err(|error| StorageError::driver(StorageCapability::Sql, OPERATION, error))?
.map_err(|error| StorageError::driver(StorageCapability::Sql, OPERATION, error))?;
crate::ensure_request_read_active(OPERATION)?;
Ok(result)
}
_ = crate::wait_for_request_read_cancellation() => {
stopped.store(true, Ordering::SeqCst);
if tokio::time::timeout(crate::sqlite_interrupt_grace_from_env(), &mut worker)
.await
.is_err()
{
worker.abort();
}
stop_on_drop.armed = false;
Err(StorageError::Timeout { operation: OPERATION.into() })
}
}
}
fn inspect_pool(pool: &ConnectionPool) -> PoolInspection {
let conn = match pool.open_standalone_writer_untracked() {
Ok(conn) => conn,
Err(e) => {
let reason = format!("guarded standalone open refused: {e}");
return PoolInspection {
checkpoint_probe: None,
checkpoint_probe_error: Some(reason.clone()),
checkpoint_pin: checkpoint_pin_diagnostics(pool, None, Some(&reason)),
size_composition: None,
size_composition_error: Some(reason.clone()),
page_accounting: None,
page_accounting_error: Some(reason.clone()),
graph_edge_integrity: None,
graph_edge_integrity_error: Some(reason.clone()),
fts_segments: None,
fts_segments_error: Some(reason),
};
}
};
let probe_result = checkpoint_probe(&conn);
let (run_status, run_age) = record_diagnostic_checkpoint_probe(pool, &probe_result);
let (checkpoint_probe, checkpoint_probe_error) = match probe_result {
Ok(probe) => (Some(probe), None),
Err(e) => (
None,
Some(format!("PRAGMA wal_checkpoint(PASSIVE) failed: {e}")),
),
};
let checkpoint_pin = checkpoint_pin_diagnostics_for_run(
checkpoint_probe.as_ref(),
checkpoint_probe_error.as_deref(),
run_status,
run_age,
);
let (graph_edge_integrity, graph_edge_integrity_error) = match graph_edge_integrity(&conn) {
Ok(integrity) => (Some(integrity), None),
Err(e) => (
None,
Some(format!("graph-edge integrity query failed: {e}")),
),
};
let (page_accounting, page_accounting_error) =
split_page_accounting_result(database_page_accounting(&conn));
let (size_composition, size_composition_error) = match database_size_composition(&conn) {
Ok(composition) => (Some(composition), None),
Err(error) => (
None,
Some(format!("database size composition query failed: {error}")),
),
};
let (fts_segments, fts_segments_error) =
split_fts_segments_result(crate::fts_maintenance::inspect_fts_segments(&conn));
PoolInspection {
checkpoint_probe,
checkpoint_probe_error,
checkpoint_pin,
size_composition,
size_composition_error,
page_accounting,
page_accounting_error,
graph_edge_integrity,
graph_edge_integrity_error,
fts_segments,
fts_segments_error,
}
}
#[cfg(test)]
#[path = "diagnostics_tests.rs"]
mod tests;