use std::path::{Path, PathBuf};
use std::time::Duration;
use super::{CheckResult, CheckStatus};
const PROBE_TIMEOUT: Duration = Duration::from_secs(10);
pub fn check_fastembed_cache() -> CheckResult {
let cache = trusty_common::embedder::resolve_fastembed_cache_dir();
let label = "fastembed cache".to_string();
if !cache.exists() {
return CheckResult::fail(
label,
format!(
"missing: {} — run `trusty-memory setup` to pre-warm",
cache.display()
),
);
}
if !cache.is_dir() {
return CheckResult::fail(label, format!("not a directory: {}", cache.display()));
}
match fastembed_cache_has_models(&cache) {
Ok(true) => CheckResult::pass(label, format!("ready at {}", cache.display())),
Ok(false) => CheckResult::warn(
label,
format!(
"{} exists but is empty — daemon will download on first request",
cache.display()
),
),
Err(e) => CheckResult::fail(label, format!("cannot read {}: {e}", cache.display())),
}
}
pub fn fastembed_cache_has_models(path: &Path) -> std::io::Result<bool> {
let mut iter = std::fs::read_dir(path)?;
Ok(iter.next().is_some())
}
#[cfg(target_os = "macos")]
pub fn check_launchd_plist() -> CheckResult {
let label = "launchd plist".to_string();
let Some(home) = dirs::home_dir() else {
return CheckResult::fail(label, "could not resolve $HOME".to_string());
};
let plist = home
.join("Library")
.join("LaunchAgents")
.join(format!("{}.plist", crate::commands::service::LAUNCHD_LABEL));
if !plist.exists() {
return CheckResult::fail(
label,
format!(
"missing: {} — run `trusty-memory service install`",
plist.display()
),
);
}
match plist_contains_fastembed_cache_path(&plist) {
Ok(true) => CheckResult::pass(label, format!("{} ok", plist.display())),
Ok(false) => CheckResult::fail(
label,
format!(
"{} is missing FASTEMBED_CACHE_PATH — reinstall via `trusty-memory service install`",
plist.display()
),
),
Err(e) => CheckResult::fail(label, format!("cannot read {}: {e}", plist.display())),
}
}
#[cfg(target_os = "macos")]
pub fn plist_contains_fastembed_cache_path(path: &Path) -> std::io::Result<bool> {
let contents = std::fs::read_to_string(path)?;
Ok(contents.contains("FASTEMBED_CACHE_PATH"))
}
pub async fn check_daemon_health() -> CheckResult {
let label = "daemon socket".to_string();
let socket = match crate::transport::uds::socket_path() {
Ok(path) => path,
Err(e) => {
return CheckResult::fail(
label,
format!("could not resolve the trusty-memory data directory: {e:#}"),
)
}
};
check_daemon_health_at(label, &socket, PROBE_TIMEOUT).await
}
pub(super) async fn check_daemon_health_at(
label: String,
socket: &Path,
timeout: Duration,
) -> CheckResult {
let request = serde_json::json!({
"jsonrpc": "2.0",
"id": 1,
"method": crate::transport::uds::METHOD_HEALTH,
"params": {},
});
let response: trusty_common::uds::server::RpcResponse =
match trusty_common::uds::send_framed_request(socket, &request, timeout).await {
Ok(response) => response,
Err(trusty_common::uds::UdsRpcError::Timeout { .. }) => {
return CheckResult::unknown(
label,
format!(
"{} did not answer {} within {timeout:?}. The connection was not \
refused, so the daemon may be alive and slow, or wedged — health \
could not be determined. Re-run when load subsides rather than \
restarting anything.",
socket.display(),
crate::transport::uds::METHOD_HEALTH,
),
);
}
Err(e) => {
return CheckResult::fail(
label,
format!(
"no daemon is serving {} — it is unreachable ({e}). \
Start it with `trusty-memory service start`.",
socket.display()
),
);
}
};
let url = socket.display().to_string();
match (response.result, response.error) {
(Some(body), _) => interpret_health_body(label, &url, 200, Some(&body)),
(None, Some(e)) => CheckResult::fail(
label,
format!(
"{} refused the health call: {} ({})",
url, e.message, e.code
),
),
(None, None) => CheckResult::unknown(
label,
format!("{url} answered with neither a result nor an error"),
),
}
}
pub(super) fn interpret_health_body(
label: String,
url: &str,
status: u16,
body: Option<&serde_json::Value>,
) -> CheckResult {
let Some(body) = body else {
return CheckResult::unknown(
label,
format!(
"{url} → {status}, but the response body could not be read or parsed. The \
listener is up; whether its workers are making progress is UNKNOWN."
),
);
};
let worker = body.get("worker");
let wedged = worker
.and_then(|w| w.get("wedged"))
.and_then(serde_json::Value::as_bool);
let oldest = worker
.and_then(|w| w.get("oldest_age_secs"))
.and_then(serde_json::Value::as_u64);
let in_flight = worker
.and_then(|w| w.get("in_flight"))
.and_then(serde_json::Value::as_u64);
let stalled_lock = worker.and_then(|w| w.get("stalled_lock"));
let wedged_reason = worker
.and_then(|w| w.get("wedged_reason"))
.and_then(serde_json::Value::as_str);
let lock_wedge = wedged_reason.map_or_else(|| stalled_lock.is_some(), |r| r == "lock");
match wedged {
Some(true) if lock_wedge && stalled_lock.is_some() => {
let field = |k: &str| stalled_lock.and_then(|s| s.get(k));
let text = |k: &str| {
field(k)
.and_then(serde_json::Value::as_str)
.unwrap_or("unknown")
};
return CheckResult::fail(
label,
format!(
"{url} → {status} BUT the daemon reports a WEDGED palace: {} lock of \
palace {} has been unavailable for at least {}s. Writes to it time out. \
Inspect with a thread sample before restarting.",
text("lock"),
text("palace"),
field("age_secs")
.and_then(serde_json::Value::as_u64)
.unwrap_or_default()
),
);
}
Some(true) => {
return CheckResult::fail(
label,
format!(
"{url} → {status} BUT the daemon reports a WEDGED worker pool: oldest \
in-flight palace operation has been running {}s with {} in flight. The \
HTTP listener answering does not mean writes are progressing (issue \
#3992). Inspect with a thread sample before restarting.",
oldest.unwrap_or_default(),
in_flight.unwrap_or_default()
),
);
}
None => {
return CheckResult::unknown(
label,
format!(
"{url} → {status}, but this daemon does not report worker-pool occupancy \
(pre-#4001 build). Liveness is confirmed; whether workers are making \
progress is UNKNOWN. Upgrade the daemon to get a real answer."
),
);
}
Some(false) => {}
}
match worker
.and_then(|w| w.get("stall_tracking_ok"))
.and_then(serde_json::Value::as_bool)
{
Some(true) => {}
Some(false) => {
let detail = body
.get("detail")
.and_then(|v| v.as_str())
.unwrap_or("no detail reported");
return CheckResult::unknown(
label,
format!(
"{url} → {status}, but the daemon's palace-lock stall tracking is not \
running: {detail}. Whether a palace lock is wedged is UNKNOWN. Restart \
the daemon to restore the detector."
),
);
}
None => {
return CheckResult::unknown(
label,
format!(
"{url} → {status}, but this daemon has no palace-lock stall detector \
(pre-#4001 build). It reports worker-pool occupancy only, and the \
2026-09-13 wedge held a lock no pool gauge could see, so whether a \
palace lock is wedged is UNKNOWN. Upgrade the daemon."
),
);
}
}
let daemon_state = body.get("daemon_state").and_then(|v| v.as_str());
let reported = body.get("status").and_then(|v| v.as_str());
if daemon_state == Some("warming") {
return CheckResult::warn(
label,
format!(
"{url} → {status}, daemon is WARMING UP (embedder still initialising). This is \
normal shortly after a restart — recall falls back to the non-embedder path \
until it finishes."
),
);
}
if reported == Some("degraded") {
let detail = body
.get("detail")
.and_then(|v| v.as_str())
.unwrap_or("no detail reported");
return CheckResult::warn(
label,
format!("{url} → {status}, daemon reports DEGRADED: {detail}"),
);
}
let occupancy = match (in_flight, oldest) {
(Some(n), Some(secs)) => format!(", {n} in flight, oldest {secs}s"),
(Some(n), None) => format!(", {n} in flight"),
_ => String::new(),
};
CheckResult::pass(
label,
format!("{url} → {status}, workers progressing{occupancy}"),
)
}
pub(super) struct DoctorSummary {
pub(super) line: String,
pub(super) healthy: bool,
}
pub(super) fn summarize(results: &[CheckResult]) -> DoctorSummary {
let count = |status: CheckStatus| results.iter().filter(|r| r.status == status).count();
let (passed, warned) = (count(CheckStatus::Pass), count(CheckStatus::Warn));
let (unknown, failed) = (count(CheckStatus::Unknown), count(CheckStatus::Fail));
DoctorSummary {
line: format!(
"{passed} passed, {warned} warnings, {unknown} undetermined, {failed} failed."
),
healthy: failed == 0 && unknown == 0,
}
}
#[cfg(test)]
#[path = "checks_tests.rs"]
mod checks_tests;
pub fn check_stale_palace_locks() -> CheckResult {
let label = "palace locks".to_string();
let data_dir = match trusty_common::resolve_data_dir("trusty-memory") {
Ok(d) => d,
Err(e) => return CheckResult::fail(label, format!("could not resolve data dir: {e}")),
};
let root = crate::resolve_palace_registry_dir(data_dir);
let locks = find_lock_files(&root);
if locks.is_empty() {
CheckResult::pass(label, format!("{} clean", root.display()))
} else {
let preview = locks
.iter()
.take(3)
.map(|p| p.display().to_string())
.collect::<Vec<_>>()
.join(", ");
let suffix = if locks.len() > 3 {
format!(" (+{} more)", locks.len() - 3)
} else {
String::new()
};
CheckResult::warn(
label,
format!(
"{} lock file(s) found: {preview}{suffix} — if the daemon is stopped, these can be removed",
locks.len()
),
)
}
}
pub fn find_lock_files(root: &Path) -> Vec<PathBuf> {
let mut out = Vec::new();
let Ok(entries) = std::fs::read_dir(root) else {
return out;
};
for entry in entries.flatten() {
let path = entry.path();
if is_lock_file(&path) {
out.push(path.clone());
}
if path.is_dir() {
if let Ok(sub) = std::fs::read_dir(&path) {
for child in sub.flatten() {
let cpath = child.path();
if is_lock_file(&cpath) {
out.push(cpath);
}
}
}
}
}
out
}
pub(super) fn is_lock_file(path: &Path) -> bool {
path.extension().and_then(|s| s.to_str()) == Some("lock")
}
const KG_REDB_WARN_BYTES: u64 = 100 * 1024 * 1024;
const KG_REDB_FAIL_BYTES: u64 = 500 * 1024 * 1024;
pub fn check_kg_redb_size() -> CheckResult {
let label = "kg.redb size".to_string();
let data_dir = match trusty_common::resolve_data_dir("trusty-memory") {
Ok(d) => d,
Err(e) => return CheckResult::fail(label, format!("could not resolve data dir: {e}")),
};
let root = crate::resolve_palace_registry_dir(data_dir);
let Some((palace, bytes)) = largest_kg_redb(&root) else {
return CheckResult::pass(label, format!("no palace store under {}", root.display()));
};
kg_redb_verdict(label, &palace, bytes)
}
pub fn largest_kg_redb(root: &Path) -> Option<(String, u64)> {
let mut best: Option<(String, u64)> = None;
for entry in std::fs::read_dir(root).ok()?.flatten() {
let dir = entry.path();
if !dir.is_dir() {
continue;
}
let Ok(meta) = std::fs::metadata(dir.join("kg.redb")) else {
continue;
};
let name = entry.file_name().to_string_lossy().into_owned();
if best.as_ref().is_none_or(|(_, b)| meta.len() > *b) {
best = Some((name, meta.len()));
}
}
best
}
pub fn kg_redb_verdict(label: String, palace: &str, bytes: u64) -> CheckResult {
let mb = bytes / (1024 * 1024);
if bytes >= KG_REDB_FAIL_BYTES {
CheckResult::fail(
label,
format!(
"palace '{palace}' kg.redb is {mb} MB — run `trusty-memory palace compact \
{palace}` (reads are still correct; this is disk, not data)"
),
)
} else if bytes >= KG_REDB_WARN_BYTES {
CheckResult::warn(
label,
format!(
"palace '{palace}' kg.redb is {mb} MB — consider `trusty-memory palace stats \
{palace}` to see what is reclaimable"
),
)
} else {
CheckResult::pass(label, format!("largest is '{palace}' at {mb} MB"))
}
}