use std::io;
use std::path::Path;
use chrono::{DateTime, Utc};
use second_brain_core::api_token;
use second_brain_core::config::{CONFIG_FILE, Config};
use second_brain_sync::peers::{PEERS_FILE, Peer, configured_peers};
use serde_json::Value;
use super::fixes;
use super::report::{Check, Context, Status};
use crate::api;
const DAEMON: &str = "daemon";
const TOKEN: &str = "token";
const DISTILLER: &str = "distiller";
const PEERS: &str = "peers";
const BOOTSTRAP: &str = "pairing";
const RECOVERY: &str = "recovery";
const IDENTITY: &str = "identity";
const JOIN_FIX: &str = "run `sb pair` on a device that syncs, then `sb pair <code>` on this device";
const RESTART: &str = "sb restart";
const WAIT_FOR_START: &str = "wait for the daemon to finish starting, then rerun `sb doctor`";
const SERVING_FIXES: &[(&str, &str)] = &[
(
"waits for disk space",
"free disk space on the filesystem named above (compaction resumes by itself)",
),
(
"store maintenance is running",
"wait for store maintenance to finish (queued writes are applied after it)",
),
("last store repair failed", "sb doctor --repair"),
(
"watch limit reached",
"raise the watch limit with `sudo sysctl fs.inotify.max_user_watches=524288`",
),
("transcript watcher", RESTART),
("capture is", RESTART),
("ingest has failed", RESTART),
("index drain", RESTART),
("search indexes unavailable", RESTART),
("to be indexed", RESTART),
("write queue not draining", RESTART),
("store compaction failed", RESTART),
];
const SYNC_PROBLEM: &str = "no sync has succeeded";
const PROGRESS_FAILURE: &str = "cannot check sync and distillation";
pub(crate) trait Api {
fn get(&self, path: &str, token: Option<&str>) -> Result<(u16, serde_json::Value), String>;
}
pub(crate) fn checks(ctx: &Context, api: &dyn Api) -> Vec<Check> {
let health = Health::fetch(api);
let token = read_token(&ctx.data_dir);
let verified = match (&health, &token) {
(Health::Down(_), _) => Verification::Unasked("the daemon is not answering"),
(Health::Starting(_), _) => {
Verification::Unasked("the daemon is still starting and refuses requests until then")
}
(_, Token::Present(token)) => Verification::Answered(
api.get("/v1/sync/status", Some(token.as_str()))
.map(|(status, _)| status),
),
_ => Verification::Unasked("there is no token to send"),
};
let mut checks = vec![
daemon_check(&health),
token_check(&ctx.data_dir, &token, verified),
distiller_check(&ctx.data_dir, health.body()),
peers_check(
&ctx.data_dir,
health.body(),
&paired_devices(api, &health, &token).unwrap_or_default(),
),
identity_check(health.body()),
recovery_check(api, &health, &token),
];
checks.extend(bootstrap_check(health.body()));
checks
}
fn bootstrap_check(health: Option<&Value>) -> Option<Check> {
(health?["bootstrap_pending"] == Value::Bool(true)).then(|| {
Check::warn(
BOOTSTRAP,
"this device waits to join a sync identity and ingests nothing until its first \
sync from the device that paired it finishes",
"if it has not joined yet, run `sb pair <code> --host <host>` with the code `sb pair` \
shows on your other device, or `sb pair --stop-waiting` to start alone; if it joined, \
make that device reachable over ssh from here, or run `sb pair --stop-waiting` to \
ingest without its snapshot",
)
})
}
pub(crate) fn daemon_answered(checks: &[Check]) -> bool {
checks
.iter()
.any(|check| check.name == DAEMON && check.status != Status::Fail)
}
enum Health {
Down(String),
Status(u16),
Starting(String),
Up(Value),
}
impl Health {
fn fetch(api: &dyn Api) -> Self {
match api.get("/health", None) {
Err(e) => Health::Down(e),
Ok((200, body)) => match api::starting_reason(&body) {
Some(reason) => Health::Starting(reason.to_string()),
None => Health::Up(body),
},
Ok((status, _)) => Health::Status(status),
}
}
fn body(&self) -> Option<&Value> {
match self {
Health::Up(body) => Some(body),
_ => None,
}
}
}
fn daemon_check(health: &Health) -> Check {
match health {
Health::Down(e) => Check::fail(
DAEMON,
format!("the daemon is not answering ({e}), so nothing is captured or recalled"),
"`sb start`, or `sb setup` if it was never installed",
),
Health::Status(status) => Check::fail(
DAEMON,
format!("the daemon answered /health with HTTP {status}, so it is up but not serving"),
RESTART,
),
Health::Starting(reason) => Check::warn(
DAEMON,
format!(
"the daemon is starting and serves no capture or recall until it finishes: {reason}"
),
WAIT_FOR_START,
),
Health::Up(body) => serving_check(body),
}
}
fn serving_check(body: &Value) -> Check {
let version = body["version"].as_str().unwrap_or("an unknown version");
let verdict = if body["serving"].is_object() {
&body["serving"]
} else {
body
};
match verdict["state"].as_str() {
None | Some("ok") => Check::ok(DAEMON, format!("running {version}")),
Some(state) => {
let reason = verdict["reason"].as_str().unwrap_or("no reason given");
let detail = format!(
"running {version} but {state}, so capture or recall is impaired: {reason}"
);
let fix = serving_fix(reason);
if state == "failing" {
Check::fail(DAEMON, detail, fix)
} else {
Check::warn(DAEMON, detail, fix)
}
}
}
}
fn serving_fix(reason: &str) -> String {
let mut fixes: Vec<&str> = Vec::new();
for &(needle, fix) in SERVING_FIXES {
if reason.contains(needle) && !fixes.contains(&fix) {
fixes.push(fix);
}
}
if fixes.is_empty() {
fixes.push(RESTART);
}
fixes.join(", then ")
}
enum Token {
Present(String),
Missing,
Empty,
Unreadable(io::Error),
}
fn read_token(data_dir: &Path) -> Token {
match api_token::read(data_dir) {
Ok(token) => Token::Present(token),
Err(_) => match std::fs::read_to_string(api_token::path(data_dir)) {
Err(e) if e.kind() == io::ErrorKind::NotFound => Token::Missing,
Err(e) => Token::Unreadable(e),
Ok(_) => Token::Empty,
},
}
}
enum Verification {
Unasked(&'static str),
Answered(Result<u16, String>),
}
fn token_check(data_dir: &Path, token: &Token, verified: Verification) -> Check {
let path = api_token::path(data_dir);
let path = path.display();
let unusable = |problem: String| {
Check::fail(
TOKEN,
format!(
"{problem}, so the CLI cannot authenticate to the daemon (the daemon writes it when it starts)"
),
RESTART,
)
};
match (token, verified) {
(Token::Missing, _) => unusable(format!("{path} is missing")),
(Token::Empty, _) => unusable(format!("{path} is empty")),
(Token::Unreadable(e), _) => Check::fail(
TOKEN,
format!("cannot read {path} ({e}), so the CLI cannot authenticate to the daemon"),
token_file_fix(&api_token::path(data_dir), e),
),
(Token::Present(_), Verification::Unasked(why)) => Check::ok(
TOKEN,
format!("{path} is present but unverified because {why}"),
),
(Token::Present(_), Verification::Answered(Ok(401))) => Check::fail(
TOKEN,
format!(
"the daemon rejects {path} because it holds a different token (the file was likely replaced while it ran), so CLI calls that need the token fail"
),
RESTART,
),
(Token::Present(_), Verification::Answered(Ok(200))) => {
Check::ok(TOKEN, "accepted by the daemon")
}
(Token::Present(_), Verification::Answered(Ok(status))) => Check::ok(
TOKEN,
format!("{path} is present but unverified: /v1/sync/status answered HTTP {status}"),
),
(Token::Present(_), Verification::Answered(Err(e))) => Check::ok(
TOKEN,
format!("{path} is present but unverified: /v1/sync/status failed ({e})"),
),
}
}
fn token_file_fix(path: &Path, error: &io::Error) -> String {
let shown = path.display();
match error.kind() {
io::ErrorKind::PermissionDenied => format!("{}, then `sb restart`", own_file_fix(path)),
io::ErrorKind::InvalidData => {
format!("remove {shown} and run `sb restart` so the daemon writes a new token")
}
_ => format!(
"make {shown} a readable file, or remove it and run `sb restart` so the daemon writes a new token"
),
}
}
fn own_file_fix(path: &Path) -> String {
fixes::reclaim_private_file(path)
}
fn identity_check(health: Option<&Value>) -> Check {
let Some(health) = health else {
return Check::skip(IDENTITY, "unasked because the daemon is not serving");
};
let identity = &health["identity"];
if identity.is_null() {
return Check::skip(IDENTITY, "the daemon predates identity reporting");
}
if identity["checked_at"].is_null() {
return Check::skip(IDENTITY, "the daemon has not read its sync identity yet");
}
let Some(tag) = identity["identity"].as_str() else {
let peers = health["sync"]["peers"].as_u64().is_some_and(|n| n > 0);
return if peers {
Check::warn(
IDENTITY,
"this device holds no sync identity key, so it cannot sync with its peers",
JOIN_FIX,
)
} else {
Check::ok(IDENTITY, "none; this device does not sync")
};
};
if identity["removed"] == true {
return Check::warn(
IDENTITY,
"another device removed this device from its sync identity, so its peers refuse it",
"join another device's identity: run `sb pair` there, then `sb pair <code>` on this \
device",
);
}
if identity["row_accepted"] == false {
return Check::warn(
IDENTITY,
"no peer has accepted this device's row, so none wraps it a newer identity key",
JOIN_FIX,
);
}
if identity["possibly_revoked"] == true {
return Check::warn(
IDENTITY,
"a peer holds identity key epochs it did not wrap to this device, which may have \
been removed",
"run `sb device list` on a device that syncs to see whether this one was removed",
);
}
let epochs = identity["epochs"].as_array().map_or(0, Vec::len);
let storage = identity["key_storage"].as_str().unwrap_or("unknown");
Check::ok(
IDENTITY,
format!("{tag}, {epochs} key epochs held, device key kept in {storage}"),
)
}
fn recovery_check(api: &dyn Api, health: &Health, token: &Token) -> Check {
let (Some(_), Token::Present(token)) = (health.body(), token) else {
return Check::skip(RECOVERY, "unasked because the daemon is not serving");
};
let body = match api.get("/v1/keys/recovery", Some(token.as_str())) {
Ok((200, body)) => body,
Ok((404, _)) => return Check::skip(RECOVERY, "the daemon predates recovery phrases"),
Ok((status, _)) => {
return Check::skip(
RECOVERY,
format!("/v1/keys/recovery answered HTTP {status}"),
);
}
Err(e) => return Check::skip(RECOVERY, format!("/v1/keys/recovery failed ({e})")),
};
if body["identity"].is_null() {
return Check::skip(RECOVERY, "this device has no sync identity yet");
}
let prints: Vec<&str> = body["fingerprints"]
.as_array()
.map(|a| a.iter().filter_map(Value::as_str).collect())
.unwrap_or_default();
if !prints.is_empty() {
return Check::ok(
RECOVERY,
format!("keys are wrapped to recovery phrase {}", prints.join(", ")),
);
}
if body["offered"].as_bool() == Some(true) {
return Check::ok(
RECOVERY,
"no recovery phrase, declined when offered; `sb recovery new` makes one",
);
}
Check::warn(
RECOVERY,
"no recovery phrase, so losing every device loses the memory",
"sb recovery new",
)
}
fn distiller_check(data_dir: &Path, health: Option<&Value>) -> Check {
let config = data_dir.join(CONFIG_FILE);
let endpoint = match llm_endpoint(&config) {
Ok(endpoint) => endpoint,
Err(e) => {
return Check::fail(
DISTILLER,
format!(
"{} does not load ({e}), so the daemon runs on defaults, or will at its next start, yet counts an LLM as configured and raw memories wait for a distiller that never runs",
config.display()
),
format!(
"correct or remove the offending section in {}, then `sb restart`",
config.display()
),
);
}
};
let reason = health.and_then(|body| body["reason"].as_str());
if let Some(failure) = reason.and_then(progress_failure) {
return Check::warn(
DISTILLER,
format!(
"the daemon {failure}, so it cannot tell whether raw memories are being distilled"
),
"`sb restart`, then `sb doctor --repair` if it persists",
);
}
if not_checked_yet(health, "distill") {
return Check::ok(
DISTILLER,
config_detail(
"distillation not checked yet by the daemon",
&config,
endpoint.as_deref(),
),
);
}
let distill_mode = health.and_then(|body| body["distill"]["mode"].as_str());
if let (Some("no-gpu"), Some(endpoint)) = (distill_mode, &endpoint) {
return Check::warn(
DISTILLER,
format!(
"the daemon runs in no-GPU mode because it has not picked up the [llm] section with endpoint {endpoint} in {} yet",
config.display()
),
RESTART,
);
}
if distill_mode.is_some_and(|mode| mode != "llm") {
return Check::ok(
DISTILLER,
"no-GPU mode: sessions are kept as agent notes and raw chunks with no LLM; add an [llm] section to config.toml to distill them",
);
}
if let Some(overdue) = reason.and_then(distill_problem) {
return Check::warn(
DISTILLER,
format!(
"{overdue}: raws are distilled only when `sb consolidate` runs, so nothing has run it lately or its LLM endpoint is down"
),
backlog_fix(&config, endpoint.as_deref()),
);
}
let detail = match distill_mode {
Some(_) => llm_detail(&config, endpoint.as_deref()),
None => config_detail(
"the daemon did not report its distill mode",
&config,
endpoint.as_deref(),
),
};
Check::ok(DISTILLER, detail)
}
fn not_checked_yet(health: Option<&Value>, report: &str) -> bool {
health.is_some_and(|body| body[report].is_object() && body[report]["checked_at"].is_null())
}
fn config_detail(unverified: &str, config: &Path, endpoint: Option<&str>) -> String {
let config = config.display();
match endpoint {
Some(endpoint) => format!(
"{unverified}; {config} names the [llm] endpoint {endpoint}, so it should distill with it"
),
None => format!(
"{unverified}; {config} has no [llm] section, so it runs in no-GPU mode unless an LLM distilled a note in the past 7 days"
),
}
}
fn llm_endpoint(config: &Path) -> Result<Option<String>, String> {
let loaded = if config.exists() {
Config::load(config).map_err(|e| format!("{e:#}"))?
} else {
Config::default()
};
Ok(loaded.llm.map(|llm| llm.endpoint))
}
fn backlog_fix(config: &Path, endpoint: Option<&str>) -> String {
let config = config.display();
match endpoint {
Some(endpoint) => format!(
"run `sb consolidate` (it uses the [llm] endpoint {endpoint} in {config}) and schedule `sb daily-pipeline` to keep up; if it fails, check that {endpoint} is reachable"
),
None => format!(
"run `sb consolidate` on the device that distills, or here with `--llm-url <endpoint>` or an [llm] section added to {config}; the daemon leaves LLM mode 7 days after the last LLM-distilled note, synced ones included, so a device that does not distill can set `alert.undistilled_overdue_hours = 0` in {config}; `sb restart` helps only if the [llm] section was removed after the daemon started"
),
}
}
fn llm_detail(config: &Path, endpoint: Option<&str>) -> String {
match endpoint {
Some(endpoint) => format!(
"LLM mode: raws are distilled by `sb consolidate` with the [llm] endpoint {endpoint}"
),
None => format!(
"LLM mode: raws are distilled by `sb consolidate`, which needs an [llm] section in {} or `--llm-url`; an LLM distilled a note in the last 7 days, synced ones included, or the daemon started while the section was there (`sb restart` re-reads the file)",
config.display()
),
}
}
fn paired_devices(api: &dyn Api, health: &Health, token: &Token) -> Option<Vec<String>> {
let (Some(_), Token::Present(token)) = (health.body(), token) else {
return None;
};
let Ok((200, body)) = api.get("/v1/devices", Some(token.as_str())) else {
return None;
};
let devices = body["devices"].as_array()?;
Some(
devices
.iter()
.filter(|d| d["this_device"] == false && d["revoked"] == false)
.filter_map(|d| {
d["name"]
.as_str()
.filter(|name| !name.is_empty())
.or(d["machine_id"].as_str())
.map(str::to_string)
})
.collect(),
)
}
fn peers_check(data_dir: &Path, health: Option<&Value>, paired: &[String]) -> Check {
let peers = match configured_peers(data_dir) {
Ok(peers) => peers.unwrap_or_default(),
Err(e) if !paired.is_empty() => {
let path = data_dir.join(PEERS_FILE);
return Check::warn(
PEERS,
format!(
"cannot read {} ({e}); the daemon still syncs with {}, but not with \
the hosts listed there",
path.display(),
paired.join(" and ")
),
peers_file_fix(&path, &e),
);
}
Err(e) => {
let path = data_dir.join(PEERS_FILE);
return Check::fail(
PEERS,
format!(
"cannot read {} ({e}), so sync cannot reach its peers",
path.display()
),
peers_file_fix(&path, &e),
);
}
};
if !paired.is_empty() {
if let Some(problem) = health.and_then(sync_problem) {
return Check::warn(
PEERS,
problem,
format!(
"check that {} is on and its sshd reachable from here; the daemon keeps \
retrying, its log names each failure as `sync with <device> failed`, and \
this clears once a sync succeeds",
paired.join(" or ")
),
);
}
return Check::ok(
PEERS,
format!(
"{} paired device{}, {}",
paired.len(),
if paired.len() == 1 { "" } else { "s" },
last_sync(health)
),
);
}
if peers.is_empty() {
return match health.and_then(sync_problem) {
Some(problem) => Check::warn(PEERS, problem, no_peers_fix(data_dir)),
None => Check::ok(PEERS, "no paired devices"),
};
}
if let Some(problem) = health.and_then(sync_problem) {
let host = Peer::parse(&peers[0]).host;
return Check::warn(
PEERS,
problem,
format!(
"run `sb sync {host}` to see the error, and check that `ssh {host}` works without a password"
),
);
}
Check::ok(
PEERS,
format!("{} configured, {}", counted(peers.len()), last_sync(health)),
)
}
fn no_peers_fix(data_dir: &Path) -> String {
format!(
"pair this device with your others: run `sb pair` on one of them, then `sb pair <code> --host <host>` here with the code it shows, or set alert.sync_overdue_hours = 0 in {} if this device no longer syncs",
data_dir.join(CONFIG_FILE).display()
)
}
fn peers_file_fix(path: &Path, error: &io::Error) -> String {
match error.kind() {
io::ErrorKind::PermissionDenied => own_file_fix(path),
io::ErrorKind::InvalidData => format!(
"rewrite {} as UTF-8 lines of user@host, one peer per line",
path.display()
),
_ => format!(
"make {} a readable file of user@host lines; reading it failed with: {error}",
path.display()
),
}
}
fn sync_problem(health: &Value) -> Option<String> {
let reason = health["reason"].as_str()?;
let rest = &reason[reason.find(SYNC_PROBLEM)?..];
let end = rest
.match_indices("; ")
.map(|(i, _)| i)
.find(|&i| {
let next = &rest[i + 2..];
starts_distill_problem(next) || next.starts_with(PROGRESS_FAILURE)
})
.unwrap_or(rest.len());
Some(rest[..end].to_string())
}
fn distill_problem(reason: &str) -> Option<&str> {
let start = std::iter::once(0)
.chain(reason.match_indices("; ").map(|(i, _)| i + 2))
.find(|&i| starts_distill_problem(&reason[i..]))?;
let rest = &reason[start..];
Some(rest.split("; ").next().unwrap_or(rest))
}
fn progress_failure(reason: &str) -> Option<&str> {
reason.find(PROGRESS_FAILURE).map(|start| &reason[start..])
}
fn starts_distill_problem(text: &str) -> bool {
let after_count = text.trim_start_matches(|c: char| c.is_ascii_digit());
after_count.len() < text.len() && after_count.starts_with(" raw memor")
}
fn last_sync(health: Option<&Value>) -> String {
let Some(body) = health else {
return "last sync unknown while the daemon is not serving".to_string();
};
if not_checked_yet(health, "sync") {
return "last sync not checked yet by the daemon".to_string();
}
let sync = &body["sync"];
let last = [&sync["last_success_at"], &sync["last_cursor_at"]]
.into_iter()
.find_map(|at| at.as_str()?.parse::<DateTime<Utc>>().ok());
match last {
Some(at) => format!("last synced {}", at.format("%Y-%m-%d %H:%M UTC")),
None => "no sync has succeeded yet".to_string(),
}
}
fn counted(peers: usize) -> String {
if peers == 1 {
"1 peer".to_string()
} else {
format!("{peers} peers")
}
}
#[cfg(test)]
mod tests {
use std::cell::RefCell;
use std::collections::HashMap;
use std::os::unix::fs::PermissionsExt;
use std::path::Path;
use chrono::{Duration, Utc};
use serde_json::{Value, json};
use super::*;
use crate::commands::doctor::report::Status;
use crate::daemon::Os;
const TOKEN: &str = "test-token-value";
const CHECKED_AT: &str = "2026-11-01T08:05:00Z";
struct FakeApi {
responses: HashMap<&'static str, Result<(u16, Value), String>>,
down: Option<String>,
calls: RefCell<Vec<(String, Option<String>)>>,
}
impl FakeApi {
fn up(health: Value) -> Self {
let mut responses = HashMap::new();
responses.insert("/health", Ok((200, health)));
responses.insert("/v1/sync/status", Ok((200, json!({"peers": []}))));
FakeApi {
responses,
down: None,
calls: RefCell::new(Vec::new()),
}
}
fn down(error: &str) -> Self {
FakeApi {
responses: HashMap::new(),
down: Some(error.to_string()),
calls: RefCell::new(Vec::new()),
}
}
fn starting(reason: &str) -> Self {
let refusal = json!({"error": format!("the daemon is starting: {reason}")});
let health = json!({
"status": "starting",
"version": "0.12.0",
"state": "starting",
"reason": reason,
"since": "2026-11-02T08:00:00Z",
"serving": {"state": "starting", "reason": reason}
});
FakeApi::up(health)
.with("/v1/sync/status", 503, refusal.clone())
.with("/v1/health", 503, refusal)
}
fn with(mut self, path: &'static str, status: u16, body: Value) -> Self {
self.responses.insert(path, Ok((status, body)));
self
}
fn calls_to(&self, path: &str) -> Vec<Option<String>> {
self.calls
.borrow()
.iter()
.filter(|(p, _)| p == path)
.map(|(_, t)| t.clone())
.collect()
}
}
impl Api for FakeApi {
fn get(&self, path: &str, token: Option<&str>) -> Result<(u16, Value), String> {
self.calls
.borrow_mut()
.push((path.to_string(), token.map(str::to_string)));
if let Some(error) = &self.down {
return Err(error.clone());
}
self.responses
.get(path)
.cloned()
.unwrap_or(Ok((404, Value::Null)))
}
}
fn context(home: &Path) -> Context {
let data_dir = home.join(".second-brain");
std::fs::create_dir_all(&data_dir).unwrap();
Context {
os: Os::Linux,
home: home.to_path_buf(),
data_dir,
sb_bin: home.join(".cargo/bin").join("sb"),
mcp_bin: Ok(home.join(".cargo/bin/second-brain-mcp")),
claude_config_dir: None,
search_path: None,
model_cache: home.join(".cache/second-brain/models"),
}
}
fn write_token(ctx: &Context, content: &str) {
std::fs::write(ctx.data_dir.join("api.token"), content).unwrap();
}
fn healthy() -> Value {
json!({
"status": "ok",
"version": "0.12.0",
"state": "ok",
"reason": null,
"since": "2026-11-01T08:00:00Z",
"serving": {"state": "ok", "reason": null},
"sync": {
"checked_at": CHECKED_AT,
"peers": null,
"synced_before": false,
"last_success_at": null,
"last_cursor_at": null
},
"distill": {
"checked_at": CHECKED_AT,
"mode": "no-gpu",
"pending": 0,
"oldest_at": null
},
"last_error": null
})
}
fn degraded(reason: &str) -> Value {
let mut body = healthy();
body["state"] = json!("degraded");
body["reason"] = json!(reason);
body["serving"] = json!({"state": "degraded", "reason": reason});
body
}
fn failing(reason: &str) -> Value {
let mut body = healthy();
body["state"] = json!("failing");
body["reason"] = json!(reason);
body["serving"] = json!({"state": "failing", "reason": reason});
body
}
fn upkeep(reason: &str, distill: Value) -> Value {
let mut body = healthy();
body["state"] = json!("degraded");
body["reason"] = json!(reason);
body["distill"] = distill;
body
}
fn distill_clause(pending: u64, waited: &str) -> String {
format!(
"{pending} raw memories have waited {waited} to be distilled; run `sb consolidate --llm-url <endpoint>` \
or add an [llm] section, or set alert.undistilled_overdue_hours = 0 if this device does not distill"
)
}
fn running_as_root() -> bool {
unsafe { libc::getuid() == 0 }
}
fn find<'a>(checks: &'a [Check], name: &str) -> &'a Check {
checks
.iter()
.find(|c| c.name == name)
.unwrap_or_else(|| panic!("no {name} check in {checks:?}"))
}
fn fix(check: &Check) -> &str {
check.fix.as_deref().unwrap_or("")
}
fn device_list(others: &[(&str, bool)]) -> Value {
let mut devices = vec![json!({
"machine_id": "m-self",
"name": "desk",
"this_device": true,
"revoked": false,
"added_by": []
})];
devices.extend(others.iter().map(|(name, revoked)| {
json!({
"machine_id": format!("m-{name}"),
"name": name,
"this_device": false,
"revoked": revoked,
"added_by": ["m-self"]
})
}));
json!({"identity": "tag", "devices": devices})
}
fn synced(at: &str) -> Value {
let mut body = healthy();
body["sync"] = json!({
"checked_at": CHECKED_AT,
"peers": null,
"synced_before": true,
"last_success_at": at,
"last_cursor_at": at
});
body
}
fn overdue() -> Value {
let reason = "no sync has succeeded in 25h 0m although this device has synced before; \
run `sb sync <user@host>` (or check the job that runs it), \
or set alert.sync_overdue_hours = 0 if this device no longer syncs";
let mut body = synced(&(Utc::now() - Duration::hours(25)).to_rfc3339());
body["state"] = json!("degraded");
body["reason"] = json!(reason);
body
}
#[test]
fn peers_are_ok_with_the_count_of_other_active_devices_in_the_device_list() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let recent = (Utc::now() - Duration::hours(1)).to_rfc3339();
let api = FakeApi::up(synced(&recent)).with(
"/v1/devices",
200,
device_list(&[("laptop", false), ("pi", false), ("old", true)]),
);
let checks = checks(&ctx, &api);
let peers = find(&checks, "peers");
assert_eq!(peers.status, Status::Ok, "{peers:?}");
assert!(peers.detail.contains("2 paired devices"), "{peers:?}");
assert!(peers.detail.contains(&recent[..10]), "{peers:?}");
assert_eq!(peers.fix, None);
}
#[test]
fn this_device_and_revoked_devices_are_not_peers() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let api = FakeApi::up(healthy()).with("/v1/devices", 200, device_list(&[("old", true)]));
let checks = checks(&ctx, &api);
let peers = find(&checks, "peers");
assert_eq!(peers.status, Status::Ok, "{peers:?}");
assert!(peers.detail.contains("no paired devices"), "{peers:?}");
}
#[test]
fn peers_warn_when_sync_with_paired_devices_is_overdue_and_name_them() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let api = FakeApi::up(overdue()).with(
"/v1/devices",
200,
device_list(&[("laptop", false), ("old", true)]),
);
let checks = checks(&ctx, &api);
let peers = find(&checks, "peers");
assert_eq!(peers.status, Status::Warn, "{peers:?}");
assert!(peers.detail.contains("no sync has succeeded"), "{peers:?}");
assert!(fix(peers).contains("laptop"), "{peers:?}");
assert!(!fix(peers).contains("old"), "{peers:?}");
assert!(!fix(peers).contains("sync-peers"), "{peers:?}");
assert!(!fix(peers).contains("sb sync"), "{peers:?}");
}
#[test]
fn a_pending_bootstrap_warns_until_the_daemon_stops_waiting() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let mut waiting = healthy();
waiting["bootstrap_pending"] = json!(true);
let before = checks(&ctx, &FakeApi::up(waiting));
let warned = find(&before, "pairing");
assert_eq!(warned.status, Status::Warn, "{warned:?}");
assert!(
fix(warned).contains("`sb pair --stop-waiting`"),
"{warned:?}"
);
let after = checks(&ctx, &FakeApi::up(healthy()));
assert!(
after.iter().all(|check| check.name != "pairing"),
"{after:?}"
);
}
#[test]
fn an_unreadable_sync_peers_only_warns_while_paired_devices_still_sync() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
std::fs::create_dir(ctx.data_dir.join("sync-peers")).unwrap();
let recent = (Utc::now() - Duration::hours(1)).to_rfc3339();
let api = FakeApi::up(synced(&recent)).with(
"/v1/devices",
200,
device_list(&[("laptop", false)]),
);
let checks = checks(&ctx, &api);
let peers = find(&checks, "peers");
assert_eq!(peers.status, Status::Warn, "{peers:?}");
assert!(
peers.detail.contains("still syncs with laptop"),
"{peers:?}"
);
}
#[test]
fn pairing_as_the_no_peers_fix_says_clears_the_peers_warning() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let before = checks(
&ctx,
&FakeApi::up(overdue()).with("/v1/devices", 200, device_list(&[])),
);
let warned = find(&before, "peers");
assert_eq!(warned.status, Status::Warn, "{warned:?}");
assert!(fix(warned).contains("sb pair"), "{warned:?}");
let recent = (Utc::now() - Duration::minutes(1)).to_rfc3339();
let after = checks(
&ctx,
&FakeApi::up(synced(&recent)).with(
"/v1/devices",
200,
device_list(&[("laptop", false)]),
),
);
let peers = find(&after, "peers");
assert_eq!(peers.status, Status::Ok, "{peers:?}");
assert!(peers.detail.contains("1 paired device"), "{peers:?}");
}
#[test]
fn the_group_reports_daemon_token_distiller_and_peers_in_that_order() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let checks = checks(&ctx, &FakeApi::up(healthy()));
let names: Vec<&str> = checks.iter().map(|c| c.name).collect();
assert_eq!(
names,
[
"daemon",
"token",
"distiller",
"peers",
"identity",
"recovery"
]
);
}
#[test]
fn health_is_fetched_once_and_sync_status_once_with_the_token() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, &format!("{TOKEN}\n"));
std::fs::write(ctx.data_dir.join("sync-peers"), "alice@laptop\n").unwrap();
std::fs::write(ctx.data_dir.join("sync.key"), "key").unwrap();
let api = FakeApi::up(healthy());
checks(&ctx, &api);
assert_eq!(api.calls_to("/health").len(), 1);
assert_eq!(api.calls_to("/v1/sync/status"), [Some(TOKEN.to_string())]);
assert_eq!(api.calls_to("/v1/devices"), [Some(TOKEN.to_string())]);
}
#[test]
fn daemon_fails_with_the_connection_error_when_nothing_answers() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let checks = checks(&ctx, &FakeApi::down("connection refused (os error 111)"));
let daemon = find(&checks, "daemon");
assert_eq!(daemon.status, Status::Fail);
assert!(daemon.detail.contains("connection refused"), "{daemon:?}");
assert!(fix(daemon).contains("sb start"), "{daemon:?}");
assert!(fix(daemon).contains("sb setup"), "{daemon:?}");
}
#[test]
fn daemon_fails_with_the_http_status_when_health_is_not_200() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let api = FakeApi::up(healthy()).with("/health", 503, Value::Null);
let checks = checks(&ctx, &api);
let daemon = find(&checks, "daemon");
assert_eq!(daemon.status, Status::Fail);
assert!(daemon.detail.contains("503"), "{daemon:?}");
assert!(fix(daemon).contains("sb restart"), "{daemon:?}");
}
#[test]
fn daemon_is_ok_with_its_version_when_healthy() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let checks = checks(&ctx, &FakeApi::up(healthy()));
let daemon = find(&checks, "daemon");
assert_eq!(daemon.status, Status::Ok);
assert!(daemon.detail.contains("running 0.12.0"), "{daemon:?}");
assert_eq!(daemon.fix, None);
}
#[test]
fn daemon_fails_with_the_health_reason_and_restarts_a_stopped_watcher() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let api = FakeApi::up(failing(
"transcript watcher is not running: No such file or directory (os error 2)",
));
let checks = checks(&ctx, &api);
let daemon = find(&checks, "daemon");
assert_eq!(daemon.status, Status::Fail, "{daemon:?}");
assert!(daemon.detail.contains("os error 2"), "{daemon:?}");
assert!(fix(daemon).contains("sb restart"), "{daemon:?}");
assert!(!fix(daemon).contains("max_user_watches"), "{daemon:?}");
}
#[test]
fn daemon_fails_when_the_watcher_hit_the_watch_limit_and_says_to_raise_it() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let api = FakeApi::up(failing(
"transcript watcher is not running: OS file watch limit reached",
));
let checks = checks(&ctx, &api);
let daemon = find(&checks, "daemon");
assert_eq!(daemon.status, Status::Fail, "{daemon:?}");
assert!(
daemon.detail.contains("OS file watch limit reached"),
"{daemon:?}"
);
assert!(fix(daemon).contains("max_user_watches"), "{daemon:?}");
assert!(fix(daemon).contains("sb restart"), "{daemon:?}");
}
#[test]
fn daemon_fails_when_capture_or_indexing_is_past_its_failing_limit() {
for reason in [
"capture is 3h 0m behind the newest transcript",
"4100 changes have waited 2h 0m to be indexed",
] {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let checks = checks(&ctx, &FakeApi::up(failing(reason)));
let daemon = find(&checks, "daemon");
assert_eq!(daemon.status, Status::Fail, "{reason}: {daemon:?}");
assert!(daemon.detail.contains(reason), "{daemon:?}");
assert!(fix(daemon).contains("sb restart"), "{daemon:?}");
}
}
#[test]
fn daemon_waiting_for_disk_space_says_to_free_space_rather_than_restart() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let api = FakeApi::up(degraded(
"store compaction waits for disk space: it needs 900 MB free at /data and has 100 MB",
));
let checks = checks(&ctx, &api);
let daemon = find(&checks, "daemon");
assert_eq!(daemon.status, Status::Warn);
assert!(daemon.detail.contains("900 MB"), "{daemon:?}");
assert!(fix(daemon).to_lowercase().contains("free"), "{daemon:?}");
assert!(!fix(daemon).contains("sb restart"), "{daemon:?}");
}
#[test]
fn a_sync_only_degradation_warns_under_peers_not_daemon() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
std::fs::write(ctx.data_dir.join("sync-peers"), "alice@laptop\n").unwrap();
std::fs::write(ctx.data_dir.join("sync.key"), "key").unwrap();
let reason = "no sync has succeeded in 25h 0m with the 1 device this device syncs with; \
run `sb doctor` for what to check";
let stale = (Utc::now() - Duration::hours(25)).to_rfc3339();
let mut body = healthy();
body["state"] = json!("degraded");
body["reason"] = json!(reason);
body["sync"] = json!({
"checked_at": CHECKED_AT,
"peers": 1,
"synced_before": true,
"last_success_at": stale,
"last_cursor_at": stale
});
let checks = checks(&ctx, &FakeApi::up(body));
assert_eq!(find(&checks, "daemon").status, Status::Ok, "{checks:?}");
assert_eq!(find(&checks, "peers").status, Status::Warn, "{checks:?}");
}
const STARTING: &str = "rebuilding search indexes for 23594 memories, 212s so far";
fn starting_home() -> (tempfile::TempDir, Context) {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
std::fs::write(ctx.data_dir.join("sync-peers"), "alice@laptop\n").unwrap();
std::fs::write(ctx.data_dir.join("sync.key"), "key").unwrap();
(home, ctx)
}
#[test]
fn a_starting_daemon_warns_with_its_reason_and_says_to_wait_and_rerun_doctor() {
let (_home, ctx) = starting_home();
let api = FakeApi::starting(STARTING);
let checks = checks(&ctx, &api);
let daemon = find(&checks, "daemon");
assert_eq!(daemon.status, Status::Warn, "{daemon:?}");
assert!(daemon.detail.contains(STARTING), "{daemon:?}");
assert!(daemon.detail.contains("starting"), "{daemon:?}");
assert!(!daemon.detail.contains("impaired"), "{daemon:?}");
assert!(fix(daemon).contains("wait"), "{daemon:?}");
assert!(fix(daemon).contains("sb doctor"), "{daemon:?}");
assert!(!fix(daemon).contains("sb restart"), "{daemon:?}");
assert!(
checks.iter().all(|c| c.status != Status::Fail),
"{checks:?}"
);
}
#[test]
fn a_starting_daemon_is_not_asked_to_verify_the_token() {
let (_home, ctx) = starting_home();
let api = FakeApi::starting(STARTING);
let checks = checks(&ctx, &api);
let token = find(&checks, "token");
assert_eq!(token.status, Status::Ok, "{token:?}");
assert!(token.detail.contains("unverified"), "{token:?}");
assert!(token.detail.contains("starting"), "{token:?}");
assert!(!token.detail.contains("503"), "{token:?}");
assert!(api.calls_to("/v1/sync/status").is_empty());
assert!(api.calls_to("/v1/devices").is_empty());
let peers = find(&checks, "peers");
assert!(!peers.detail.contains("not answering"), "{peers:?}");
}
#[test]
fn a_daemon_that_finished_starting_clears_the_starting_warning() {
let (_home, ctx) = starting_home();
let before = checks(&ctx, &FakeApi::starting(STARTING));
assert_eq!(find(&before, "daemon").status, Status::Warn, "{before:?}");
let after = checks(&ctx, &FakeApi::up(healthy()));
let daemon = find(&after, "daemon");
assert_eq!(daemon.status, Status::Ok, "{daemon:?}");
assert_eq!(daemon.fix, None);
let token = find(&after, "token");
assert_eq!(token.detail, "accepted by the daemon", "{token:?}");
}
#[test]
fn token_fails_when_the_file_is_missing() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
let checks = checks(&ctx, &FakeApi::up(healthy()));
let token = find(&checks, "token");
assert_eq!(token.status, Status::Fail);
assert!(token.detail.contains("api.token"), "{token:?}");
assert!(fix(token).contains("sb restart"), "{token:?}");
}
#[test]
fn token_fails_when_the_file_is_empty() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, " \n");
let checks = checks(&ctx, &FakeApi::up(healthy()));
let token = find(&checks, "token");
assert_eq!(token.status, Status::Fail);
assert!(token.detail.contains("empty"), "{token:?}");
assert!(fix(token).contains("sb restart"), "{token:?}");
}
#[test]
fn token_fails_when_the_daemon_rejects_it() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let api =
FakeApi::up(healthy()).with("/v1/sync/status", 401, json!({"error": "unauthorized"}));
let checks = checks(&ctx, &api);
let token = find(&checks, "token");
assert_eq!(token.status, Status::Fail);
assert!(
token.detail.to_lowercase().contains("different"),
"{token:?}"
);
assert!(fix(token).contains("sb restart"), "{token:?}");
}
#[test]
fn token_is_ok_when_the_daemon_accepts_it() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let checks = checks(&ctx, &FakeApi::up(healthy()));
let token = find(&checks, "token");
assert_eq!(token.status, Status::Ok, "{token:?}");
assert_eq!(token.fix, None);
}
#[test]
fn token_is_present_but_unverified_when_the_daemon_is_down() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let checks = checks(&ctx, &FakeApi::down("connection refused"));
let token = find(&checks, "token");
assert_eq!(token.status, Status::Ok, "{token:?}");
assert!(token.detail.contains("unverified"), "{token:?}");
assert_eq!(token.fix, None);
}
#[test]
fn distiller_fails_with_the_parse_error_when_config_is_broken() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
std::fs::write(
ctx.data_dir.join("config.toml"),
"[llm]\nendpoint = \"http://gpu.invalid:8081\"\n",
)
.unwrap();
let checks = checks(&ctx, &FakeApi::up(healthy()));
let distiller = find(&checks, "distiller");
assert_eq!(distiller.status, Status::Fail);
assert!(distiller.detail.contains("model"), "{distiller:?}");
assert!(
distiller.detail.contains("or will at its next start"),
"{distiller:?}"
);
assert!(fix(distiller).contains("config.toml"), "{distiller:?}");
}
#[test]
fn distiller_warns_when_the_daemon_has_not_picked_up_a_new_llm_section() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
std::fs::write(
ctx.data_dir.join("config.toml"),
"[llm]\nendpoint = \"http://gpu.invalid:8081\"\nmodel = \"qwen\"\n",
)
.unwrap();
let checks = checks(&ctx, &FakeApi::up(healthy()));
let distiller = find(&checks, "distiller");
assert_eq!(distiller.status, Status::Warn, "{distiller:?}");
assert!(distiller.detail.contains("[llm]"), "{distiller:?}");
assert!(distiller.detail.contains("picked up"), "{distiller:?}");
assert!(fix(distiller).contains("sb restart"), "{distiller:?}");
}
#[test]
fn distiller_reports_a_failed_store_read_with_a_local_fix_whether_or_not_peers_exist() {
let failure = "cannot check sync and distillation for 2h 0m: \
reading raws pending distillation: database is locked";
for with_peers in [false, true] {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
if with_peers {
std::fs::write(ctx.data_dir.join("sync-peers"), "alice@laptop\n").unwrap();
std::fs::write(ctx.data_dir.join("sync.key"), "key").unwrap();
}
let body = upkeep(
failure,
json!({"checked_at": CHECKED_AT, "mode": "llm", "pending": 0, "oldest_at": null}),
);
let checks = checks(&ctx, &FakeApi::up(body));
let distiller = find(&checks, "distiller");
assert_eq!(distiller.status, Status::Warn, "{distiller:?}");
assert!(
distiller.detail.contains("database is locked"),
"{distiller:?}"
);
assert!(fix(distiller).contains("sb restart"), "{distiller:?}");
assert!(
fix(distiller).contains("sb doctor --repair"),
"{distiller:?}"
);
let peers = find(&checks, "peers");
assert_eq!(peers.status, Status::Ok, "peers={with_peers}: {peers:?}");
assert!(!fix(peers).contains("ssh"), "{peers:?}");
}
}
#[test]
fn distiller_in_no_gpu_mode_is_ok_and_says_how_to_turn_on_an_llm() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let checks = checks(&ctx, &FakeApi::up(healthy()));
let distiller = find(&checks, "distiller");
assert_eq!(distiller.status, Status::Ok);
assert!(distiller.detail.contains("agent notes"), "{distiller:?}");
assert!(distiller.detail.contains("[llm]"), "{distiller:?}");
assert_eq!(distiller.fix, None);
}
#[test]
fn distiller_warns_when_an_llm_backlog_is_older_than_a_day() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let reason = format!(
"no sync has succeeded in 25h 0m with the 1 device this device syncs with; \
run `sb doctor` for what to check; {}",
distill_clause(520, "2d")
);
let body = upkeep(
&reason,
json!({
"checked_at": CHECKED_AT,
"mode": "llm",
"pending": 520,
"oldest_at": (Utc::now() - Duration::hours(48)).to_rfc3339()
}),
);
let checks = checks(&ctx, &FakeApi::up(body));
let distiller = find(&checks, "distiller");
assert_eq!(distiller.status, Status::Warn);
assert!(
distiller
.detail
.contains("520 raw memories have waited 2d to be distilled"),
"{distiller:?}"
);
assert!(
!distiller.detail.contains("no sync has succeeded"),
"{distiller:?}"
);
assert!(fix(distiller).contains("[llm]"), "{distiller:?}");
assert!(fix(distiller).contains("endpoint"), "{distiller:?}");
assert!(fix(distiller).contains("sb consolidate"), "{distiller:?}");
}
#[test]
fn distiller_follows_the_daemons_undistilled_verdict_not_the_config_window() {
let cases = [
(0, Some(distill_clause(9, "8h 0m")), 8, Status::Warn),
(6, None, 8, Status::Ok),
(24, None, 480, Status::Ok),
];
for (hours, reason, waited, expected) in cases {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
std::fs::write(
ctx.data_dir.join("config.toml"),
format!("[alert]\nundistilled_overdue_hours = {hours}\n"),
)
.unwrap();
let distill = json!({
"checked_at": CHECKED_AT,
"mode": "llm",
"pending": 9,
"oldest_at": (Utc::now() - Duration::hours(waited)).to_rfc3339()
});
let body = match &reason {
Some(reason) => upkeep(reason, distill),
None => {
let mut body = healthy();
body["distill"] = distill;
body
}
};
let checks = checks(&ctx, &FakeApi::up(body));
let distiller = find(&checks, "distiller");
assert_eq!(
distiller.status, expected,
"{hours}h config, reason {reason:?}: {distiller:?}"
);
if expected == Status::Warn {
assert!(
distiller
.detail
.contains("9 raw memories have waited 8h 0m to be distilled"),
"{distiller:?}"
);
}
}
}
#[test]
fn distiller_is_ok_when_the_llm_backlog_is_recent() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let mut body = healthy();
body["distill"] = json!({
"checked_at": CHECKED_AT,
"mode": "llm",
"pending": 3,
"oldest_at": (Utc::now() - Duration::hours(1)).to_rfc3339()
});
let checks = checks(&ctx, &FakeApi::up(body));
let distiller = find(&checks, "distiller");
assert_eq!(distiller.status, Status::Ok, "{distiller:?}");
assert_eq!(distiller.fix, None);
}
#[test]
fn distiller_in_llm_mode_names_sb_consolidate_as_what_distills() {
for config in [
"[llm]\nendpoint = \"http://gpu.invalid:8081\"\nmodel = \"qwen\"\n",
"",
] {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
std::fs::write(ctx.data_dir.join("config.toml"), config).unwrap();
let mut body = healthy();
body["distill"] = json!({
"checked_at": CHECKED_AT,
"mode": "llm",
"pending": 3,
"oldest_at": (Utc::now() - Duration::hours(1)).to_rfc3339()
});
let checks = checks(&ctx, &FakeApi::up(body));
let distiller = find(&checks, "distiller");
assert_eq!(distiller.status, Status::Ok, "{distiller:?}");
assert!(distiller.detail.contains("sb consolidate"), "{distiller:?}");
if config.is_empty() {
let detail = &distiller.detail;
assert!(detail.contains("--llm-url"), "{distiller:?}");
assert!(detail.contains("[llm] section"), "{distiller:?}");
assert!(detail.contains("7 days"), "{distiller:?}");
assert!(detail.contains("sb restart"), "{distiller:?}");
}
}
}
#[test]
fn llm_backlog_without_an_llm_section_names_the_7_day_window_and_the_alert_off_switch() {
for config in [None, Some("")] {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
if let Some(config) = config {
std::fs::write(ctx.data_dir.join("config.toml"), config).unwrap();
}
let body = upkeep(
&distill_clause(40, "2d"),
json!({
"checked_at": CHECKED_AT,
"mode": "llm",
"pending": 40,
"oldest_at": (Utc::now() - Duration::hours(48)).to_rfc3339()
}),
);
let checks = checks(&ctx, &FakeApi::up(body));
let distiller = find(&checks, "distiller");
assert_eq!(distiller.status, Status::Warn, "{config:?}: {distiller:?}");
let fix = fix(distiller);
assert!(fix.contains("sb consolidate"), "{config:?}: {distiller:?}");
assert!(fix.contains("--llm-url"), "{config:?}: {distiller:?}");
assert!(fix.contains("7 days"), "{config:?}: {distiller:?}");
assert!(
fix.contains("alert.undistilled_overdue_hours = 0"),
"{config:?}: {distiller:?}"
);
assert!(fix.contains("sb restart"), "{config:?}: {distiller:?}");
assert!(
!fix.contains("settles into no-GPU mode"),
"a restart does not leave LLM mode inside the 7-day window: {distiller:?}"
);
}
}
#[test]
fn distiller_reads_the_llm_endpoint_from_config_when_the_daemon_is_down() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
std::fs::write(
ctx.data_dir.join("config.toml"),
"[llm]\nendpoint = \"http://gpu.invalid:8081\"\nmodel = \"qwen\"\n",
)
.unwrap();
let checks = checks(&ctx, &FakeApi::down("connection refused"));
let distiller = find(&checks, "distiller");
assert_eq!(distiller.status, Status::Ok, "{distiller:?}");
assert!(
distiller.detail.contains("http://gpu.invalid:8081"),
"{distiller:?}"
);
}
#[test]
fn distiller_without_config_or_daemon_is_no_gpu() {
let mut no_distill = healthy();
no_distill.as_object_mut().unwrap().remove("distill");
let apis = [
("down", FakeApi::down("connection refused")),
(
"503",
FakeApi::up(healthy()).with("/health", 503, Value::Null),
),
("no distill report", FakeApi::up(no_distill)),
];
for (case, api) in apis {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let checks = checks(&ctx, &api);
let distiller = find(&checks, "distiller");
assert_eq!(distiller.status, Status::Ok, "{case}: {distiller:?}");
assert!(distiller.detail.contains("[llm]"), "{case}: {distiller:?}");
assert!(
distiller.detail.contains("no-GPU mode"),
"{case}: {distiller:?}"
);
assert!(
distiller
.detail
.contains("unless an LLM distilled a note in the past 7 days"),
"{case}: {distiller:?}"
);
assert!(
!distiller.detail.starts_with("no-GPU mode:"),
"{case}: {distiller:?}"
);
assert!(
!distiller.detail.contains("LLM mode"),
"{case}: {distiller:?}"
);
}
}
#[test]
fn peers_are_none_configured_without_a_sync_peers_file() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let checks = checks(&ctx, &FakeApi::up(healthy()));
let peers = find(&checks, "peers");
assert_eq!(peers.status, Status::Ok);
assert!(peers.detail.contains("no paired devices"), "{peers:?}");
}
#[test]
fn peers_are_none_configured_when_sync_peers_has_only_comments() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
std::fs::write(ctx.data_dir.join("sync-peers"), "# alice@laptop\n\n").unwrap();
let checks = checks(&ctx, &FakeApi::up(healthy()));
let peers = find(&checks, "peers");
assert_eq!(peers.status, Status::Ok);
assert!(peers.detail.contains("no paired devices"), "{peers:?}");
}
#[test]
fn unreadable_peers_file_gives_chown_and_chmod_with_the_quoted_path() {
if running_as_root() {
return;
}
let home = tempfile::tempdir().unwrap();
let ctx = context(&home.path().join("my home"));
write_token(&ctx, TOKEN);
let path = ctx.data_dir.join("sync-peers");
std::fs::write(&path, "alice@laptop\n").unwrap();
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o000)).unwrap();
let checks = checks(&ctx, &FakeApi::up(healthy()));
let peers = find(&checks, "peers");
let quoted = crate::daemon::shell_word(&path.to_string_lossy());
assert!(quoted.starts_with('\''), "{quoted}");
let uid = unsafe { libc::getuid() };
assert_eq!(peers.status, Status::Fail, "{peers:?}");
assert!(fix(peers).contains("chown "), "{peers:?}");
assert!(fix(peers).contains(&format!("{uid} {quoted}")), "{peers:?}");
assert!(
fix(peers).contains(&format!("chmod 600 {quoted}")),
"{peers:?}"
);
}
#[test]
fn non_utf8_peers_file_says_to_rewrite_it_as_user_at_host_lines() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
std::fs::write(ctx.data_dir.join("sync-peers"), [0xff, 0xfe, b'\n']).unwrap();
let checks = checks(&ctx, &FakeApi::up(healthy()));
let peers = find(&checks, "peers");
assert_eq!(peers.status, Status::Fail, "{peers:?}");
assert!(fix(peers).contains("UTF-8"), "{peers:?}");
assert!(fix(peers).contains("user@host"), "{peers:?}");
assert!(!fix(peers).contains("chmod"), "{peers:?}");
}
#[test]
fn other_peers_file_errors_name_the_path_and_the_error_in_the_fix() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let path = ctx.data_dir.join("sync-peers");
std::fs::create_dir(&path).unwrap();
let error = std::fs::read_to_string(&path).unwrap_err().to_string();
let checks = checks(&ctx, &FakeApi::up(healthy()));
let peers = find(&checks, "peers");
assert_eq!(peers.status, Status::Fail, "{peers:?}");
assert!(
fix(peers).contains(&path.display().to_string()),
"{peers:?}"
);
assert!(fix(peers).contains(&error), "{error}: {peers:?}");
assert!(!fix(peers).contains("chmod"), "{peers:?}");
}
#[test]
fn peers_pass_without_a_sync_key_once_the_daemon_holds_the_key() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
std::fs::write(ctx.data_dir.join("sync-peers"), "alice@laptop\n").unwrap();
let checks = checks(&ctx, &FakeApi::up(healthy()));
let peers = find(&checks, "peers");
assert_eq!(peers.status, Status::Ok, "{peers:?}");
}
#[test]
fn peers_warn_when_sync_is_overdue_and_name_the_first_peer_host() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
std::fs::write(
ctx.data_dir.join("sync-peers"),
"alice@laptop /home/alice/.cargo/bin/sb\nbob@desk\n",
)
.unwrap();
std::fs::write(ctx.data_dir.join("sync.key"), "key").unwrap();
let reason = "no sync has succeeded in 25h 0m with the 2 devices this device syncs with; \
run `sb doctor` for what to check";
let stale = (Utc::now() - Duration::hours(25)).to_rfc3339();
let mut body = degraded(reason);
body["serving"] = json!({"state": "ok", "reason": null});
body["sync"] = json!({
"checked_at": CHECKED_AT,
"peers": 2,
"synced_before": true,
"last_success_at": stale,
"last_cursor_at": stale
});
let checks = checks(&ctx, &FakeApi::up(body));
let peers = find(&checks, "peers");
assert_eq!(peers.status, Status::Warn);
assert!(peers.detail.contains("no sync has succeeded"), "{peers:?}");
assert!(fix(peers).contains("sb sync alice@laptop"), "{peers:?}");
assert!(fix(peers).contains("ssh alice@laptop"), "{peers:?}");
assert!(!fix(peers).contains("/home/alice"), "{peers:?}");
}
#[test]
fn peers_are_ok_with_the_count_and_last_success_when_sync_is_current() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
std::fs::write(ctx.data_dir.join("sync-peers"), "alice@laptop\nbob@desk\n").unwrap();
std::fs::write(ctx.data_dir.join("sync.key"), "key").unwrap();
let recent = (Utc::now() - Duration::hours(1)).to_rfc3339();
let mut body = healthy();
body["sync"] = json!({
"checked_at": CHECKED_AT,
"peers": 2,
"synced_before": true,
"last_success_at": recent,
"last_cursor_at": recent
});
let checks = checks(&ctx, &FakeApi::up(body));
let peers = find(&checks, "peers");
assert_eq!(peers.status, Status::Ok, "{peers:?}");
assert!(peers.detail.contains("2 peers"), "{peers:?}");
assert!(peers.detail.contains(&recent[..10]), "{peers:?}");
assert_eq!(peers.fix, None);
}
#[test]
fn peers_warn_with_the_daemons_clause_when_a_device_that_synced_lists_no_peers() {
let empty_file = "no sync has succeeded in 25h 0m, and ~/.second-brain/sync-peers lists no peers \
although this device has synced before; add each peer to it as user@host, \
one per line, or set alert.sync_overdue_hours = 0 if this device no longer syncs";
let no_file = "no sync has succeeded in 25h 0m although this device has synced before; \
run `sb sync <user@host>` (or check the job that runs it), \
or set alert.sync_overdue_hours = 0 if this device no longer syncs";
for (peers_file, peers, reason) in [
(Some("# alice@laptop\n\n"), json!(0), empty_file),
(None, Value::Null, no_file),
] {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
if let Some(content) = peers_file {
std::fs::write(ctx.data_dir.join("sync-peers"), content).unwrap();
}
let stale = (Utc::now() - Duration::hours(25)).to_rfc3339();
let full_reason = format!("{reason}; {}", distill_clause(4, "2d"));
let mut body = upkeep(
&full_reason,
json!({
"checked_at": CHECKED_AT,
"mode": "llm",
"pending": 4,
"oldest_at": (Utc::now() - Duration::hours(48)).to_rfc3339()
}),
);
body["sync"] = json!({
"checked_at": CHECKED_AT,
"peers": peers,
"synced_before": true,
"last_success_at": stale,
"last_cursor_at": stale
});
let checks = checks(&ctx, &FakeApi::up(body));
let found = find(&checks, "peers");
assert_eq!(found.status, Status::Warn, "{peers_file:?}: {found:?}");
assert!(
found.detail.contains("no sync has succeeded in 25h 0m"),
"{found:?}"
);
assert!(!found.detail.contains("raw memor"), "{found:?}");
assert!(!found.detail.contains("no paired devices"), "{found:?}");
assert!(fix(found).contains("sb pair"), "{found:?}");
assert!(!fix(found).contains("sync-peers"), "{found:?}");
assert!(!fix(found).contains("user@host"), "{found:?}");
assert!(
fix(found).contains("alert.sync_overdue_hours = 0"),
"{found:?}"
);
assert!(!fix(found).contains("ssh"), "{found:?}");
}
}
#[test]
fn distiller_falls_back_to_config_before_the_daemons_first_check() {
let endpoint = "http://gpu.invalid:8081";
for config in [None, Some(endpoint)] {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
if let Some(endpoint) = config {
std::fs::write(
ctx.data_dir.join("config.toml"),
format!("[llm]\nendpoint = \"{endpoint}\"\nmodel = \"qwen\"\n"),
)
.unwrap();
}
let mut body = healthy();
body["distill"] = json!({
"checked_at": null,
"mode": "llm",
"pending": 0,
"oldest_at": null
});
let checks = checks(&ctx, &FakeApi::up(body));
let distiller = find(&checks, "distiller");
assert_eq!(distiller.status, Status::Ok, "{config:?}: {distiller:?}");
assert!(
distiller.detail.contains("not checked yet"),
"{config:?}: {distiller:?}"
);
match config {
Some(endpoint) => assert!(distiller.detail.contains(endpoint), "{distiller:?}"),
None => {
assert!(!distiller.detail.contains("LLM mode"), "{distiller:?}");
assert!(
!distiller
.detail
.contains("distilled notes in the last 7 days"),
"{distiller:?}"
);
assert!(distiller.detail.contains("[llm]"), "{distiller:?}");
}
}
}
}
#[test]
fn peers_say_the_last_sync_is_not_checked_yet_before_the_daemons_first_check() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
std::fs::write(ctx.data_dir.join("sync-peers"), "alice@laptop\n").unwrap();
std::fs::write(ctx.data_dir.join("sync.key"), "key").unwrap();
let mut body = healthy();
body["sync"] = json!({
"checked_at": null,
"peers": null,
"synced_before": false,
"last_success_at": null,
"last_cursor_at": null
});
let checks = checks(&ctx, &FakeApi::up(body));
let peers = find(&checks, "peers");
assert_eq!(peers.status, Status::Ok, "{peers:?}");
assert!(peers.detail.contains("1 peer"), "{peers:?}");
assert!(peers.detail.contains("not checked yet"), "{peers:?}");
assert!(!peers.detail.contains("no sync has succeeded"), "{peers:?}");
}
#[test]
fn peers_say_no_sync_has_succeeded_once_the_daemon_has_checked() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
std::fs::write(ctx.data_dir.join("sync-peers"), "alice@laptop\n").unwrap();
std::fs::write(ctx.data_dir.join("sync.key"), "key").unwrap();
let mut body = healthy();
body["sync"]["peers"] = json!(1);
let checks = checks(&ctx, &FakeApi::up(body));
let peers = find(&checks, "peers");
assert_eq!(peers.status, Status::Ok, "{peers:?}");
assert!(
peers.detail.contains("no sync has succeeded yet"),
"{peers:?}"
);
assert!(!peers.detail.contains("not checked yet"), "{peers:?}");
}
#[test]
fn non_utf8_token_says_to_remove_it_and_restart_so_the_daemon_writes_a_new_one() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
let path = ctx.data_dir.join("api.token");
std::fs::write(&path, [0xff, 0xfe, b'\n']).unwrap();
let checks = checks(&ctx, &FakeApi::up(healthy()));
let token = find(&checks, "token");
assert_eq!(token.status, Status::Fail, "{token:?}");
assert!(fix(token).contains("sb restart"), "{token:?}");
assert!(fix(token).contains("remove"), "{token:?}");
assert!(
fix(token).contains(&path.display().to_string()),
"{token:?}"
);
assert!(!fix(token).contains("user the daemon runs as"), "{token:?}");
}
#[test]
fn unreadable_token_gives_the_peers_files_chown_and_chmod_then_restart() {
if running_as_root() {
return;
}
let home = tempfile::tempdir().unwrap();
let ctx = context(&home.path().join("my home"));
write_token(&ctx, TOKEN);
let path = ctx.data_dir.join("api.token");
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o000)).unwrap();
let checks = checks(&ctx, &FakeApi::up(healthy()));
let token = find(&checks, "token");
let quoted = crate::daemon::shell_word(&path.to_string_lossy());
assert!(quoted.starts_with('\''), "{quoted}");
let uid = unsafe { libc::getuid() };
let permission_fix =
peers_file_fix(&path, &io::Error::from(io::ErrorKind::PermissionDenied));
assert_eq!(token.status, Status::Fail, "{token:?}");
assert!(fix(token).contains(&permission_fix), "{token:?}");
assert!(
fix(token).contains(&format!("chown {uid} {quoted}")),
"{token:?}"
);
assert!(
fix(token).contains(&format!("chmod 600 {quoted}")),
"{token:?}"
);
assert!(fix(token).contains("sb restart"), "{token:?}");
assert!(!fix(token).contains("||"), "{token:?}");
assert!(!fix(token).contains("user the daemon runs as"), "{token:?}");
assert!(!fix(token).contains("remove"), "{token:?}");
}
#[test]
fn daemon_answered_is_true_only_when_the_daemon_check_is_present_and_not_failing() {
let cases = [
(vec![Check::ok(DAEMON, "running 0.12.0")], true),
(
vec![Check::warn(DAEMON, "running but degraded", RESTART)],
true,
),
(
vec![Check::fail(DAEMON, "not answering", "sb start")],
false,
),
(
vec![Check::ok(super::TOKEN, "accepted by the daemon")],
false,
),
(Vec::new(), false),
];
for (checks, expected) in cases {
assert_eq!(daemon_answered(&checks), expected, "{checks:?}");
}
}
#[test]
fn the_identity_check_warns_with_a_fix_that_clears_it() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let paired = json!({
"checked_at": CHECKED_AT,
"key_storage": "tpm",
"identity": "m-0001",
"epochs": [0, 1],
"removed": false,
"possibly_revoked": false,
"row_accepted": true,
"peers": []
});
let check_with_peers = |identity: Value, peers: Value| {
let mut health = healthy();
health["identity"] = identity;
health["sync"]["peers"] = peers;
find(&checks(&ctx, &FakeApi::up(health)), "identity").clone()
};
let check_with = |identity: Value| check_with_peers(identity, Value::Null);
let with = |changes: Value| {
let mut identity = paired.clone();
for (key, value) in changes.as_object().unwrap() {
identity[key] = value.clone();
}
identity
};
let ok = check_with(paired.clone());
assert_eq!(ok.status, Status::Ok, "{ok:?}");
assert!(
ok.detail.contains("m-0001") && ok.detail.contains("tpm"),
"{ok:?}"
);
let keyless = with(json!({"identity": null, "epochs": [], "row_accepted": null}));
assert_eq!(
check_with(keyless.clone()).status,
Status::Ok,
"peers unknown"
);
assert_eq!(
check_with_peers(keyless.clone(), json!(0)).status,
Status::Ok,
"no peers"
);
std::fs::write(ctx.data_dir.join("sync-peers"), "alice@laptop\n").unwrap();
assert_eq!(
check_with_peers(keyless.clone(), json!(0)).status,
Status::Ok,
"the daemon dials no peer, whatever sync-peers says"
);
let unpaired = check_with_peers(keyless, json!(1));
assert_eq!(unpaired.status, Status::Warn, "{unpaired:?}");
assert_eq!(fix(&unpaired), JOIN_FIX);
let unaccepted = check_with(with(json!({"row_accepted": false})));
assert_eq!(unaccepted.status, Status::Warn, "{unaccepted:?}");
assert_eq!(fix(&unaccepted), JOIN_FIX);
let withheld = check_with(with(json!({"possibly_revoked": true})));
assert_eq!(withheld.status, Status::Warn, "{withheld:?}");
assert!(fix(&withheld).contains("sb device list"), "{withheld:?}");
let removed = check_with(with(json!({"removed": true, "possibly_revoked": true})));
assert_eq!(removed.status, Status::Warn, "{removed:?}");
assert!(fix(&removed).contains("sb pair <code>"), "{removed:?}");
assert_eq!(check_with(paired).status, Status::Ok, "after the fix");
assert_eq!(check_with(Value::Null).status, Status::Skip, "old daemon");
let down = checks(&ctx, &FakeApi::down("connection refused"));
assert_eq!(find(&down, "identity").status, Status::Skip);
}
#[test]
fn the_recovery_check_warns_until_a_phrase_exists_or_was_declined() {
let home = tempfile::tempdir().unwrap();
let ctx = context(home.path());
write_token(&ctx, TOKEN);
let state = |prints: Value, offered: bool| json!({"identity": "m-tag", "fingerprints": prints, "offered": offered});
let check_with = |body: Value| {
let api = FakeApi::up(healthy()).with("/v1/keys/recovery", 200, body);
let checks = checks(&ctx, &api);
let found = find(&checks, "recovery").clone();
assert_eq!(api.calls_to("/v1/keys/recovery"), [Some(TOKEN.to_string())]);
found
};
let missing = check_with(state(json!([]), false));
assert_eq!(missing.status, Status::Warn, "{missing:?}");
assert_eq!(fix(&missing), "sb recovery new");
let after_fix = check_with(state(json!(["0123456789abcdef"]), true));
assert_eq!(after_fix.status, Status::Ok, "{after_fix:?}");
assert!(after_fix.detail.contains("0123456789abcdef"));
let declined = check_with(state(json!([]), true));
assert_eq!(declined.status, Status::Ok, "{declined:?}");
let lone = check_with(json!({"identity": null, "fingerprints": [], "offered": false}));
assert_eq!(lone.status, Status::Skip, "{lone:?}");
let old = checks(&ctx, &FakeApi::up(healthy()));
assert_eq!(find(&old, "recovery").status, Status::Skip);
let down = checks(&ctx, &FakeApi::down("connection refused"));
assert_eq!(find(&down, "recovery").status, Status::Skip);
}
}