use std::collections::BTreeMap;
use std::fs;
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use meerkat_core::{
REALM_MANIFEST_FILE_NAME, SessionCheckpointMetadataState, SessionId,
session_checkpoint_metadata_state,
};
use rusqlite::OptionalExtension;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use crate::error::StoreError;
pub const REALM_SQLITE_FILES: &[&str] = &[
"sessions.sqlite3",
"runtime.sqlite3",
"workgraph.sqlite3",
"jobs.sqlite3",
"memory/memory.sqlite3",
"tasks.db",
"sessions_jsonl/session_index.sqlite3",
];
pub const FINDING_LEGACY_HOME_SESSIONS_DIR: &str = "legacy-home-sessions-dir";
pub fn enumerate_realm_sqlite_files(realm_dir: &Path) -> Vec<PathBuf> {
enumerate_realm_sqlite_inventory(realm_dir).files
}
#[non_exhaustive]
#[derive(Debug, Default)]
pub struct RealmSqliteInventory {
pub files: Vec<PathBuf>,
pub symlinks: Vec<PathBuf>,
}
pub fn enumerate_realm_sqlite_inventory(realm_dir: &Path) -> RealmSqliteInventory {
let mut inventory = RealmSqliteInventory::default();
for path in REALM_SQLITE_FILES
.iter()
.map(|relative| realm_dir.join(relative))
{
classify_inventory_path(path, &mut inventory);
}
if let Ok(entries) = fs::read_dir(realm_dir.join("mobs")) {
for entry in entries.filter_map(Result::ok) {
let path = entry.path();
if path.extension().and_then(|ext| ext.to_str()) == Some("db") {
classify_inventory_path(path, &mut inventory);
}
}
}
inventory.files.sort();
inventory.symlinks.sort();
inventory
}
fn classify_inventory_path(path: PathBuf, inventory: &mut RealmSqliteInventory) {
match fs::symlink_metadata(&path) {
Ok(metadata) if metadata.file_type().is_symlink() => inventory.symlinks.push(path),
Ok(metadata) if metadata.is_file() => inventory.files.push(path),
_ => {}
}
}
fn mob_database_files(realm_dir: &Path) -> Vec<PathBuf> {
let mut files = Vec::new();
if let Ok(entries) = fs::read_dir(realm_dir.join("mobs")) {
for entry in entries.filter_map(Result::ok) {
let path = entry.path();
if path.extension().and_then(|ext| ext.to_str()) == Some("db") && path.is_file() {
files.push(path);
}
}
}
files.sort();
files
}
pub const REALM_WRITE_ADMISSION_STEM: &str = "realm";
pub fn realm_write_admission_target(realm_dir: &Path) -> PathBuf {
realm_dir.join(REALM_WRITE_ADMISSION_STEM)
}
pub fn store_realm_admission_target(store_dir: &Path) -> Option<PathBuf> {
let parent = store_dir.parent()?;
parent
.join(REALM_MANIFEST_FILE_NAME)
.is_file()
.then(|| realm_write_admission_target(parent))
}
#[derive(Debug)]
pub struct RealmMaintenanceFence {
admission: meerkat_sqlite::ExclusiveFence,
fences: Vec<meerkat_sqlite::ExclusiveFence>,
databases: Vec<PathBuf>,
}
impl RealmMaintenanceFence {
pub fn acquire(realm_dir: &Path, deadline: Duration) -> Result<Self, StoreError> {
let started = Instant::now();
let remaining = |started: &Instant| deadline.saturating_sub(started.elapsed());
let admission = meerkat_sqlite::ExclusiveFence::acquire(
&realm_write_admission_target(realm_dir),
remaining(&started),
)
.map_err(StoreError::from)?;
let mut databases: Vec<PathBuf> = REALM_SQLITE_FILES
.iter()
.map(|relative| realm_dir.join(relative))
.collect();
databases.sort();
let mut fences = Vec::with_capacity(databases.len());
for database in &databases {
if let Some(parent) = database.parent() {
fs::create_dir_all(parent)?;
}
let fence = meerkat_sqlite::ExclusiveFence::acquire(database, remaining(&started))
.map_err(|error| {
StoreError::from(error)
})?;
fences.push(fence);
}
let mut first_pass = true;
loop {
let new: Vec<PathBuf> = mob_database_files(realm_dir)
.into_iter()
.filter(|database| !databases.contains(database))
.collect();
if new.is_empty() {
break;
}
if !first_pass && started.elapsed() >= deadline {
return Err(StoreError::Internal(format!(
"mob databases kept appearing under '{}' during maintenance-fence \
acquisition; realm is not quiescent",
realm_dir.join("mobs").display()
)));
}
first_pass = false;
for database in new {
let fence = meerkat_sqlite::ExclusiveFence::acquire(&database, remaining(&started))
.map_err(StoreError::from)?;
fences.push(fence);
databases.push(database);
}
}
databases.sort();
Ok(Self {
admission,
fences,
databases,
})
}
pub fn fenced_databases(&self) -> &[PathBuf] {
&self.databases
}
pub fn admission_lock_path(&self) -> &Path {
self.admission.lock_path()
}
pub fn len(&self) -> usize {
self.fences.len()
}
pub fn is_empty(&self) -> bool {
self.fences.is_empty()
}
}
pub fn backup_artifact_name(original: &str, purpose: &str) -> String {
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|elapsed| elapsed.as_secs())
.unwrap_or(0);
let version = env!("CARGO_PKG_VERSION");
if purpose.is_empty() {
format!("{original}.pre-{version}-{timestamp}")
} else {
format!("{original}.pre-{version}-{timestamp}.{purpose}")
}
}
pub fn is_backup_artifact_name(name: &str) -> bool {
let Some(idx) = name.rfind(".pre-") else {
return false;
};
if idx == 0 {
return false;
}
let suffix = &name[idx + ".pre-".len()..];
let Some((version, rest)) = suffix.split_once('-') else {
return false;
};
let version_ok = !version.is_empty()
&& version.split('.').count() >= 2
&& version
.split('.')
.all(|part| !part.is_empty() && part.chars().all(|c| c.is_ascii_digit()));
if !version_ok {
return false;
}
let (timestamp, purpose) = match rest.split_once('.') {
Some((timestamp, purpose)) => (timestamp, Some(purpose)),
None => (rest, None),
};
let timestamp_ok = !timestamp.is_empty() && timestamp.chars().all(|c| c.is_ascii_digit());
let purpose_ok = purpose.is_none_or(|p| {
!p.is_empty()
&& p.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_')
});
timestamp_ok && purpose_ok
}
pub fn is_quarantine_artifact_name(name: &str) -> bool {
let Some(idx) = name.rfind(".corrupt-") else {
return false;
};
if idx == 0 {
return false;
}
let timestamp = &name[idx + ".corrupt-".len()..];
!timestamp.is_empty() && timestamp.chars().all(|c| c.is_ascii_digit())
}
#[non_exhaustive]
#[derive(Debug)]
pub struct ArchivedPath {
pub archive: PathBuf,
pub warnings: Vec<String>,
}
pub fn archive_path_read_only_reported(
path: &Path,
purpose: &str,
) -> Result<ArchivedPath, StoreError> {
let name = path
.file_name()
.and_then(|name| name.to_str())
.ok_or_else(|| {
StoreError::Internal(format!(
"cannot archive '{}': path has no UTF-8 file name",
path.display()
))
})?;
let archive = path.with_file_name(backup_artifact_name(name, purpose));
fs::rename(path, &archive)?;
let mut warnings = Vec::new();
strip_write_permissions(&archive, &mut warnings);
sync_parent_dir_reported(&archive, &mut warnings);
Ok(ArchivedPath { archive, warnings })
}
pub fn archive_path_read_only(path: &Path, purpose: &str) -> Result<PathBuf, StoreError> {
archive_path_read_only_reported(path, purpose).map(|archived| archived.archive)
}
fn sync_parent_dir_reported(path: &Path, warnings: &mut Vec<String>) {
#[cfg(unix)]
{
let Some(parent) = path
.parent()
.filter(|parent| !parent.as_os_str().is_empty())
else {
return;
};
let synced = fs::File::open(parent).and_then(|dir| dir.sync_all());
if let Err(error) = synced {
warnings.push(format!(
"archive rename to '{}' is not crash-durable: fsync of parent directory '{}' \
failed: {error}",
path.display(),
parent.display()
));
}
}
#[cfg(not(unix))]
{
let _ = (path, &mut *warnings);
}
}
fn strip_write_permissions(path: &Path, warnings: &mut Vec<String>) {
let metadata = match fs::symlink_metadata(path) {
Ok(metadata) => metadata,
Err(error) => {
warnings.push(format!(
"archive hardening skipped for '{}': {error}",
path.display()
));
return;
}
};
if metadata.file_type().is_symlink() {
return;
}
if metadata.is_dir() {
match fs::read_dir(path) {
Ok(entries) => {
for entry in entries.filter_map(Result::ok) {
strip_write_permissions(&entry.path(), warnings);
}
}
Err(error) => warnings.push(format!(
"archive hardening could not list '{}': {error}",
path.display()
)),
}
}
let mut permissions = metadata.permissions();
permissions.set_readonly(true);
if let Err(error) = fs::set_permissions(path, permissions) {
warnings.push(format!(
"archive '{}' may remain writable: {error}",
path.display()
));
}
}
fn restore_write_permissions_best_effort(path: &Path) {
let Ok(metadata) = fs::symlink_metadata(path) else {
return;
};
if metadata.file_type().is_symlink() {
return;
}
#[allow(clippy::permissions_set_readonly_false)] {
let mut permissions = metadata.permissions();
permissions.set_readonly(false);
let _ = fs::set_permissions(path, permissions);
}
if metadata.is_dir()
&& let Ok(entries) = fs::read_dir(path)
{
for entry in entries.filter_map(Result::ok) {
restore_write_permissions_best_effort(&entry.path());
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MigrateMode {
#[default]
DryRun,
Apply,
}
#[non_exhaustive]
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct MigrateReport {
#[serde(default)]
pub mode: MigrateMode,
#[serde(default)]
pub swept_roots: Vec<PathBuf>,
#[serde(default)]
pub realms: Vec<RealmMigrateReport>,
#[serde(default)]
pub split_brain: Vec<SplitBrainReport>,
#[serde(default)]
pub findings: Vec<meerkat_core::StorageFinding>,
#[serde(default)]
pub errors: Vec<String>,
}
impl MigrateReport {
pub fn new(mode: MigrateMode, swept_roots: Vec<PathBuf>) -> Self {
Self {
mode,
swept_roots,
..Self::default()
}
}
pub fn has_errors(&self) -> bool {
!self.errors.is_empty() || self.realms.iter().any(|realm| !realm.errors.is_empty())
}
}
#[non_exhaustive]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RealmMigrateReport {
pub realm: String,
pub root: PathBuf,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub backend: Option<String>,
#[serde(default)]
pub ledger: Vec<LedgerBaselineEntry>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub adoption: Option<CheckpointAdoptionOutcome>,
#[serde(default)]
pub notes: Vec<String>,
#[serde(default)]
pub errors: Vec<String>,
}
impl RealmMigrateReport {
pub fn new(realm: impl Into<String>, root: PathBuf) -> Self {
Self {
realm: realm.into(),
root,
backend: None,
ledger: Vec::new(),
adoption: None,
notes: Vec::new(),
errors: Vec::new(),
}
}
}
#[non_exhaustive]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LedgerBaselineEntry {
pub database: PathBuf,
pub domain: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub before: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub after: Option<i64>,
pub action: LedgerBaselineAction,
}
impl LedgerBaselineEntry {
pub fn new(database: PathBuf, domain: impl Into<String>, action: LedgerBaselineAction) -> Self {
Self {
database,
domain: domain.into(),
before: None,
after: None,
action,
}
}
}
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum LedgerBaselineAction {
WouldStamp,
Recorded,
Stamped,
AlreadyCurrent,
ReportOnly,
}
#[non_exhaustive]
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct CheckpointAdoptionOutcome {
#[serde(default)]
pub scanned: usize,
#[serde(default)]
pub already_verified: usize,
#[serde(default)]
pub adopted: usize,
#[serde(default)]
pub legacy_pending: usize,
#[serde(default)]
pub refused: Vec<CheckpointAdoptionRefusal>,
#[serde(default)]
pub failures: Vec<(String, String)>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub skipped: Option<String>,
}
#[non_exhaustive]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CheckpointAdoptionRefusal {
pub session_id: String,
pub reason: String,
}
impl CheckpointAdoptionRefusal {
pub fn new(session_id: impl Into<String>, reason: impl Into<String>) -> Self {
Self {
session_id: session_id.into(),
reason: reason.into(),
}
}
}
#[non_exhaustive]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SplitBrainReport {
pub realm: String,
pub locations: Vec<PathBuf>,
#[serde(default)]
pub sessions_equal: usize,
#[serde(default)]
pub sessions: Vec<SessionDivergenceEntry>,
#[serde(default)]
pub files: Vec<FileDivergenceEntry>,
pub resolution: SplitBrainResolution,
#[serde(default)]
pub errors: Vec<String>,
}
impl SplitBrainReport {
pub fn new(realm: impl Into<String>, locations: Vec<PathBuf>) -> Self {
Self {
realm: realm.into(),
locations,
sessions_equal: 0,
sessions: Vec::new(),
files: Vec::new(),
resolution: SplitBrainResolution::Refused {
reason: "split-brain unresolved: rerun with `--apply --adopt-root <path>` to \
adopt one root and archive the other copies read-only"
.to_string(),
},
errors: Vec::new(),
}
}
pub fn comparison_is_conclusive(&self) -> bool {
self.errors.is_empty()
&& self
.sessions
.iter()
.all(|entry| entry.status != DivergenceStatus::Unknown)
&& self
.files
.iter()
.all(|entry| entry.status != DivergenceStatus::Unknown)
}
}
#[non_exhaustive]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SessionDivergenceEntry {
pub session_id: String,
pub status: DivergenceStatus,
}
#[non_exhaustive]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FileDivergenceEntry {
pub file: String,
pub status: DivergenceStatus,
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case", tag = "kind")]
pub enum DivergenceStatus {
Equal,
Divergent,
OnlyIn {
location: PathBuf,
},
Unknown,
}
#[non_exhaustive]
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "snake_case", tag = "kind")]
pub enum SplitBrainResolution {
Refused {
reason: String,
},
Archived {
adopted: PathBuf,
archived: Vec<PathBuf>,
},
ArchiveFailed {
adopted: PathBuf,
archived: Vec<PathBuf>,
reason: String,
},
}
#[non_exhaustive]
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct PruneReport {
#[serde(default)]
pub mode: MigrateMode,
#[serde(default)]
pub swept_roots: Vec<PathBuf>,
#[serde(default)]
pub older_than_days: u64,
#[serde(default)]
pub artifacts: Vec<PruneArtifact>,
#[serde(default)]
pub errors: Vec<String>,
}
impl PruneReport {
pub fn new(mode: MigrateMode, swept_roots: Vec<PathBuf>, older_than_days: u64) -> Self {
Self {
mode,
swept_roots,
older_than_days,
..Self::default()
}
}
}
#[non_exhaustive]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PruneArtifact {
pub path: PathBuf,
pub kind: PruneArtifactKind,
#[serde(default)]
pub bytes: u64,
#[serde(default)]
pub age_days: u64,
pub action: PruneAction,
}
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum PruneArtifactKind {
BackupArtifact,
QuarantinedIndex,
}
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum PruneAction {
WouldDelete,
Deleted,
Kept,
DeleteFailed,
}
#[non_exhaustive]
#[derive(Debug, Clone)]
pub struct RealmDirEntry {
pub realm_id: String,
pub backend: Option<String>,
pub state_root: PathBuf,
pub dir: PathBuf,
pub manifest_readable: bool,
}
pub fn list_realm_dirs(state_root: &Path) -> Vec<RealmDirEntry> {
let Ok(entries) = fs::read_dir(state_root) else {
return Vec::new();
};
let mut dirs: Vec<PathBuf> = entries
.filter_map(Result::ok)
.map(|entry| entry.path())
.filter(|path| path.is_dir())
.collect();
dirs.sort();
let mut realms = Vec::new();
for dir in dirs {
let name = dir
.file_name()
.map(|name| name.to_string_lossy().into_owned())
.unwrap_or_default();
if is_backup_artifact_name(&name) {
continue; }
let manifest_path = dir.join(REALM_MANIFEST_FILE_NAME);
if !manifest_path.is_file() {
continue;
}
let parsed = fs::read(&manifest_path)
.ok()
.and_then(|bytes| serde_json::from_slice::<serde_json::Value>(&bytes).ok())
.and_then(|value| {
let realm_id = value.get("realm_id")?.as_str()?.to_string();
let backend = value
.get("backend")
.and_then(|backend| backend.as_str())
.map(ToString::to_string);
Some((realm_id, backend))
});
let (realm_id, backend, manifest_readable) = match parsed {
Some((realm_id, backend)) => (realm_id, backend, true),
None => (name, None, false),
};
realms.push(RealmDirEntry {
realm_id,
backend,
state_root: state_root.to_path_buf(),
dir,
manifest_readable,
});
}
realms
}
pub fn read_domain_versions(db_path: &Path) -> Result<Option<Vec<(String, i64)>>, StoreError> {
let conn = meerkat_sqlite::open(db_path, meerkat_sqlite::ConnectionProfile::ReadOnly)
.map_err(StoreError::from)?;
let table_exists = conn
.query_row(
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'meerkat_schema'",
[],
|_| Ok(()),
)
.optional()?;
if table_exists.is_none() {
return Ok(None);
}
let mut statement =
conn.prepare("SELECT domain, version FROM meerkat_schema ORDER BY domain")?;
let rows = statement
.query_map([], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
})?
.collect::<Result<Vec<_>, _>>()?;
Ok(Some(rows))
}
pub fn supported_domain_version(domain: &str) -> Option<i64> {
match domain {
#[cfg(feature = "sqlite")]
"session-store" => Some(crate::sqlite_store::SESSION_STORE_DOMAIN.supported_version()),
#[cfg(feature = "sqlite")]
"schedule-store" => {
Some(crate::schedule_sqlite_store::SCHEDULE_STORE_DOMAIN.supported_version())
}
"jsonl-index" => Some(crate::index::JSONL_INDEX_DOMAIN.supported_version()),
_ => None,
}
}
#[non_exhaustive]
#[derive(Debug, Clone)]
pub struct LedgerDomainReading {
pub database: PathBuf,
pub domain: String,
pub version: Option<i64>,
}
#[non_exhaustive]
#[derive(Debug, Clone)]
pub struct FutureDomainVersion {
pub database: PathBuf,
pub domain: String,
pub found: i64,
pub supported: i64,
}
#[non_exhaustive]
#[derive(Debug, Default)]
pub struct RealmLedgerBaseline {
pub rows: Vec<LedgerDomainReading>,
pub future: Vec<FutureDomainVersion>,
pub errors: Vec<String>,
}
pub fn read_realm_ledger_baseline(realm_dir: &Path) -> RealmLedgerBaseline {
let mut baseline = RealmLedgerBaseline::default();
for (relative, expected_domains) in crate::doctor::REALM_DATABASE_FILES {
let db_path = realm_dir.join(relative);
if !db_path.is_file() {
continue;
}
let rows = match read_domain_versions(&db_path) {
Ok(rows) => rows.unwrap_or_default(),
Err(error) => {
baseline.errors.push(format!(
"ledger unreadable for {}: {error}",
db_path.display()
));
continue;
}
};
for (domain, version) in &rows {
if let Some(supported) = supported_domain_version(domain)
&& *version > supported
{
baseline.future.push(FutureDomainVersion {
database: db_path.clone(),
domain: domain.clone(),
found: *version,
supported,
});
}
baseline.rows.push(LedgerDomainReading {
database: db_path.clone(),
domain: domain.clone(),
version: Some(*version),
});
}
for expected in *expected_domains {
if !rows.iter().any(|(domain, _)| domain == expected) {
baseline.rows.push(LedgerDomainReading {
database: db_path.clone(),
domain: (*expected).to_string(),
version: None,
});
}
}
}
baseline
}
#[non_exhaustive]
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
pub struct LegacySessionCensus {
#[serde(default)]
pub verified: usize,
#[serde(default)]
pub legacy: usize,
#[serde(default)]
pub invalid: usize,
}
fn table_exists(conn: &rusqlite::Connection, table: &str) -> Result<bool, rusqlite::Error> {
Ok(conn
.query_row(
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1",
[table],
|_| Ok(()),
)
.optional()?
.is_some())
}
pub fn census_legacy_sessions(db_path: &Path) -> Result<LegacySessionCensus, StoreError> {
let conn = meerkat_sqlite::open(db_path, meerkat_sqlite::ConnectionProfile::ReadOnly)
.map_err(StoreError::from)?;
let mut census = LegacySessionCensus::default();
let mut classify = |session_id: &str, metadata_json: &str| {
let Ok(id) = SessionId::parse(session_id) else {
census.invalid += 1;
return;
};
let Ok(metadata) =
serde_json::from_str::<serde_json::Map<String, serde_json::Value>>(metadata_json)
else {
census.invalid += 1;
return;
};
match session_checkpoint_metadata_state(&id, &metadata) {
Ok(SessionCheckpointMetadataState::Stamped(_)) => census.verified += 1,
Ok(SessionCheckpointMetadataState::LegacyUnverified { .. }) => census.legacy += 1,
Err(_) => census.invalid += 1,
}
};
let heads_exist = table_exists(&conn, "session_heads")?;
if heads_exist {
let mut statement = conn.prepare("SELECT session_id, metadata_json FROM session_heads")?;
let mut rows = statement.query([])?;
while let Some(row) = rows.next()? {
let session_id: String = row.get(0)?;
let metadata_json: String = row.get(1)?;
classify(&session_id, &metadata_json);
}
}
if table_exists(&conn, "sessions")? {
let sql = if heads_exist {
"SELECT session_id, metadata_json FROM sessions \
WHERE session_id NOT IN (SELECT session_id FROM session_heads)"
} else {
"SELECT session_id, metadata_json FROM sessions"
};
let mut statement = conn.prepare(sql)?;
let mut rows = statement.query([])?;
while let Some(row) = rows.next()? {
let session_id: String = row.get(0)?;
let metadata_json: String = row.get(1)?;
classify(&session_id, &metadata_json);
}
}
Ok(census)
}
type SessionDigests = BTreeMap<String, [u8; 32]>;
fn session_digests(db_path: &Path) -> Result<SessionDigests, StoreError> {
let conn = meerkat_sqlite::open(db_path, meerkat_sqlite::ConnectionProfile::ReadOnly)
.map_err(StoreError::from)?;
let mut hashers: BTreeMap<String, Sha256> = BTreeMap::new();
let mut feed = |session_id: String, tag: &str, chunks: &[&[u8]]| {
let hasher = hashers.entry(session_id).or_default();
hasher.update(tag.as_bytes());
hasher.update([0u8]);
for chunk in chunks {
hasher.update(chunk);
hasher.update([0u8]);
}
};
if table_exists(&conn, "sessions")? {
let mut statement = conn.prepare(
"SELECT session_id, metadata_json, session_json FROM sessions ORDER BY session_id",
)?;
let mut rows = statement.query([])?;
while let Some(row) = rows.next()? {
let session_id: String = row.get(0)?;
let metadata = row
.get::<_, meerkat_sqlite::JsonColumnBytes>(1)?
.into_bytes();
let document = row
.get::<_, meerkat_sqlite::JsonColumnBytes>(2)?
.into_bytes();
feed(session_id, "sessions", &[&metadata, &document]);
}
}
if table_exists(&conn, "session_heads")? {
let mut statement = conn.prepare(
"SELECT session_id, metadata_json, head_json FROM session_heads ORDER BY session_id",
)?;
let mut rows = statement.query([])?;
while let Some(row) = rows.next()? {
let session_id: String = row.get(0)?;
let metadata = row
.get::<_, meerkat_sqlite::JsonColumnBytes>(1)?
.into_bytes();
let head = row
.get::<_, meerkat_sqlite::JsonColumnBytes>(2)?
.into_bytes();
feed(session_id, "head", &[&metadata, &head]);
}
}
if table_exists(&conn, "session_strand_messages")? {
let mut statement = conn.prepare(
"SELECT session_id, strand, seq, message_json FROM session_strand_messages \
ORDER BY session_id, strand, seq",
)?;
let mut rows = statement.query([])?;
while let Some(row) = rows.next()? {
let session_id: String = row.get(0)?;
let strand: String = row.get(1)?;
let seq: i64 = row.get(2)?;
let message = row
.get::<_, meerkat_sqlite::JsonColumnBytes>(3)?
.into_bytes();
feed(
session_id,
"message",
&[strand.as_bytes(), seq.to_be_bytes().as_slice(), &message],
);
}
}
if table_exists(&conn, "session_rewrites")? {
let mut statement = conn.prepare(
"SELECT session_id, rewrite_idx, commit_json FROM session_rewrites \
ORDER BY session_id, rewrite_idx",
)?;
let mut rows = statement.query([])?;
while let Some(row) = rows.next()? {
let session_id: String = row.get(0)?;
let index: i64 = row.get(1)?;
let commit = row
.get::<_, meerkat_sqlite::JsonColumnBytes>(2)?
.into_bytes();
feed(
session_id,
"rewrite",
&[index.to_be_bytes().as_slice(), &commit],
);
}
}
Ok(hashers
.into_iter()
.map(|(session_id, hasher)| (session_id, hasher.finalize().into()))
.collect())
}
fn stream_file_into(hasher: &mut Sha256, path: &Path) -> Result<(), StoreError> {
let mut file = fs::File::open(path)?;
std::io::copy(&mut file, hasher)?;
Ok(())
}
fn file_digest(path: &Path) -> Result<[u8; 32], StoreError> {
let mut hasher = Sha256::new();
stream_file_into(&mut hasher, path)?;
let mut wal = path.as_os_str().to_os_string();
wal.push("-wal");
let wal = PathBuf::from(wal);
match fs::symlink_metadata(&wal) {
Ok(metadata) if metadata.file_type().is_symlink() => {
return Err(StoreError::Internal(format!(
"refusing to follow symlink {} while digesting {}",
wal.display(),
path.display()
)));
}
Ok(metadata) if metadata.is_file() => stream_file_into(&mut hasher, &wal)?,
_ => {}
}
Ok(hasher.finalize().into())
}
#[derive(Debug)]
struct RealmContentSnapshot {
sessions: Option<SessionDigests>,
files: BTreeMap<String, [u8; 32]>,
unreadable: Vec<String>,
errors: Vec<String>,
}
impl RealmContentSnapshot {
fn poisons(&self, relative: &str) -> bool {
self.unreadable.iter().any(|entry| {
entry == relative || (entry.ends_with('/') && relative.starts_with(entry.as_str()))
})
}
fn record_unreadable(&mut self, relative: String, error: String) {
self.errors.push(error);
self.unreadable.push(relative);
}
}
fn snapshot_relative(location: &Path, path: &Path) -> Option<String> {
path.strip_prefix(location)
.ok()
.map(|relative| relative.to_string_lossy().replace('\\', "/"))
}
fn digest_entry_into(path: &Path, relative: String, snapshot: &mut RealmContentSnapshot) {
match fs::symlink_metadata(path) {
Ok(metadata) if metadata.file_type().is_symlink() => snapshot.record_unreadable(
relative,
format!(
"refusing to follow symlink {} during divergence comparison",
path.display()
),
),
Ok(metadata) if metadata.is_file() => match file_digest(path) {
Ok(digest) => {
snapshot.files.insert(relative, digest);
}
Err(error) => snapshot.record_unreadable(
relative,
format!("file digest unavailable for {}: {error}", path.display()),
),
},
Ok(_) => snapshot.record_unreadable(
relative,
format!(
"{} is not a regular file; divergence cannot account for it",
path.display()
),
),
Err(error) => {
snapshot
.record_unreadable(relative, format!("cannot stat {}: {error}", path.display()));
}
}
}
fn snapshot_jsonl_sessions(location: &Path, snapshot: &mut RealmContentSnapshot) {
let dir = location.join("sessions_jsonl");
let entries = match fs::read_dir(&dir) {
Ok(entries) => entries,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return,
Err(error) => {
snapshot.record_unreadable(
"sessions_jsonl/".to_string(),
format!("cannot list {}: {error}", dir.display()),
);
return;
}
};
let mut paths: Vec<PathBuf> = entries
.filter_map(Result::ok)
.map(|entry| entry.path())
.collect();
paths.sort();
for path in paths {
if path.extension().and_then(|ext| ext.to_str()) != Some("jsonl") {
continue;
}
let Some(relative) = snapshot_relative(location, &path) else {
continue;
};
digest_entry_into(&path, relative, snapshot);
}
}
fn snapshot_tree(location: &Path, tree: &str, snapshot: &mut RealmContentSnapshot) {
let root = location.join(tree);
match fs::symlink_metadata(&root) {
Ok(metadata) if metadata.file_type().is_symlink() => {
snapshot.record_unreadable(
format!("{tree}/"),
format!(
"refusing to follow symlink {} during divergence comparison",
root.display()
),
);
return;
}
Ok(metadata) if metadata.is_dir() => {}
Ok(_) | Err(_) => return, }
walk_tree_into(location, &root, snapshot);
}
fn walk_tree_into(location: &Path, dir: &Path, snapshot: &mut RealmContentSnapshot) {
let entries = match fs::read_dir(dir) {
Ok(entries) => entries,
Err(error) => {
let relative = snapshot_relative(location, dir).unwrap_or_default();
snapshot.record_unreadable(
format!("{relative}/"),
format!("cannot list {}: {error}", dir.display()),
);
return;
}
};
let mut paths: Vec<PathBuf> = entries
.filter_map(Result::ok)
.map(|entry| entry.path())
.collect();
paths.sort();
for path in paths {
let Some(relative) = snapshot_relative(location, &path) else {
continue;
};
match fs::symlink_metadata(&path) {
Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => {
walk_tree_into(location, &path, snapshot);
}
_ => digest_entry_into(&path, relative, snapshot),
}
}
}
fn snapshot_realm_content(location: &Path) -> RealmContentSnapshot {
let mut snapshot = RealmContentSnapshot {
sessions: Some(SessionDigests::new()),
files: BTreeMap::new(),
unreadable: Vec::new(),
errors: Vec::new(),
};
let sessions_db = location.join("sessions.sqlite3");
match fs::symlink_metadata(&sessions_db) {
Ok(metadata) if metadata.file_type().is_symlink() => snapshot.sessions = None,
Ok(metadata) if metadata.is_file() => match session_digests(&sessions_db) {
Ok(digests) => snapshot.sessions = Some(digests),
Err(error) => {
snapshot.sessions = None;
snapshot.errors.push(format!(
"session divergence unavailable for {}: {error}",
sessions_db.display()
));
}
},
_ => {}
}
let inventory = enumerate_realm_sqlite_inventory(location);
for link in &inventory.symlinks {
let Some(relative) = snapshot_relative(location, link) else {
continue;
};
snapshot.record_unreadable(
relative,
format!(
"refusing to follow symlink {} during divergence comparison",
link.display()
),
);
}
for db in inventory.files {
let Some(relative) = snapshot_relative(location, &db) else {
continue;
};
match file_digest(&db) {
Ok(digest) => {
snapshot.files.insert(relative, digest);
}
Err(error) => snapshot.record_unreadable(
relative,
format!("file digest unavailable for {}: {error}", db.display()),
),
}
}
snapshot_jsonl_sessions(location, &mut snapshot);
snapshot_tree(location, "blobs", &mut snapshot);
snapshot_tree(location, "artifacts", &mut snapshot);
snapshot
}
pub fn compute_split_brain_report(realm: &str, locations: &[PathBuf]) -> SplitBrainReport {
let mut report = SplitBrainReport::new(realm, locations.to_vec());
let snapshots: Vec<(PathBuf, RealmContentSnapshot)> = locations
.iter()
.map(|location| (location.clone(), snapshot_realm_content(location)))
.collect();
for (_, snapshot) in &snapshots {
report.errors.extend(snapshot.errors.iter().cloned());
}
let sessions_poisoned = snapshots
.iter()
.any(|(_, snapshot)| snapshot.sessions.is_none());
let mut all_ids: Vec<String> = snapshots
.iter()
.filter_map(|(_, snapshot)| snapshot.sessions.as_ref())
.flat_map(|digests| digests.keys().cloned())
.collect();
all_ids.sort();
all_ids.dedup();
for session_id in all_ids {
if sessions_poisoned {
report.sessions.push(SessionDivergenceEntry {
session_id,
status: DivergenceStatus::Unknown,
});
continue;
}
let holders: Vec<(&PathBuf, &[u8; 32])> = snapshots
.iter()
.filter_map(|(location, snapshot)| {
snapshot
.sessions
.as_ref()?
.get(&session_id)
.map(|digest| (location, digest))
})
.collect();
if holders.len() == 1 {
report.sessions.push(SessionDivergenceEntry {
session_id,
status: DivergenceStatus::OnlyIn {
location: holders[0].0.clone(),
},
});
} else if holders.len() == snapshots.len()
&& holders.iter().all(|(_, digest)| *digest == holders[0].1)
{
report.sessions_equal += 1;
} else {
report.sessions.push(SessionDivergenceEntry {
session_id,
status: DivergenceStatus::Divergent,
});
}
}
let mut relative_files: Vec<String> = snapshots
.iter()
.flat_map(|(_, snapshot)| {
snapshot.files.keys().cloned().chain(
snapshot
.unreadable
.iter()
.filter(|entry| !entry.ends_with('/'))
.cloned(),
)
})
.collect();
relative_files.sort();
relative_files.dedup();
for relative in relative_files {
if snapshots
.iter()
.any(|(_, snapshot)| snapshot.poisons(&relative))
{
report.files.push(FileDivergenceEntry {
file: relative,
status: DivergenceStatus::Unknown,
});
continue;
}
let holders: Vec<(&PathBuf, &[u8; 32])> = snapshots
.iter()
.filter_map(|(location, snapshot)| {
snapshot
.files
.get(&relative)
.map(|digest| (location, digest))
})
.collect();
let status = if holders.len() == 1 {
DivergenceStatus::OnlyIn {
location: holders[0].0.clone(),
}
} else if holders.len() == snapshots.len()
&& holders.iter().all(|(_, digest)| *digest == holders[0].1)
{
DivergenceStatus::Equal
} else {
DivergenceStatus::Divergent
};
report.files.push(FileDivergenceEntry {
file: relative,
status,
});
}
report
}
fn recursive_size(path: &Path) -> u64 {
let Ok(metadata) = fs::symlink_metadata(path) else {
return 0;
};
if metadata.is_dir() {
let Ok(entries) = fs::read_dir(path) else {
return 0;
};
entries
.filter_map(Result::ok)
.map(|entry| recursive_size(&entry.path()))
.sum()
} else {
metadata.len()
}
}
pub fn registered_artifact_timestamp(name: &str) -> Option<u64> {
if is_backup_artifact_name(name) {
let idx = name.rfind(".pre-")?;
let suffix = &name[idx + ".pre-".len()..];
let (_, rest) = suffix.split_once('-')?;
let timestamp = rest
.split_once('.')
.map_or(rest, |(timestamp, _)| timestamp);
return timestamp.parse().ok();
}
if is_quarantine_artifact_name(name) {
let idx = name.rfind(".corrupt-")?;
return name[idx + ".corrupt-".len()..].parse().ok();
}
None
}
fn age_days(path: &Path, name: &str) -> u64 {
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|elapsed| elapsed.as_secs())
.unwrap_or(0);
if let Some(registered) = registered_artifact_timestamp(name) {
return now.saturating_sub(registered) / 86_400;
}
fs::symlink_metadata(path)
.and_then(|metadata| metadata.modified())
.ok()
.and_then(|modified| SystemTime::now().duration_since(modified).ok())
.map(|age| age.as_secs() / 86_400)
.unwrap_or(0)
}
fn artifact_kind(name: &str) -> Option<PruneArtifactKind> {
if is_backup_artifact_name(name) {
Some(PruneArtifactKind::BackupArtifact)
} else if is_quarantine_artifact_name(name) {
Some(PruneArtifactKind::QuarantinedIndex)
} else {
None
}
}
pub fn registered_artifact_original(name: &str) -> Option<&str> {
if is_backup_artifact_name(name) {
return Some(&name[..name.rfind(".pre-")?]);
}
if is_quarantine_artifact_name(name) {
return Some(&name[..name.rfind(".corrupt-")?]);
}
None
}
fn push_artifacts_in(
dir: &Path,
dirs_too: bool,
original_filter: Option<&str>,
artifacts: &mut Vec<PruneArtifact>,
) {
let Ok(entries) = fs::read_dir(dir) else {
return;
};
let mut paths: Vec<PathBuf> = entries
.filter_map(Result::ok)
.map(|entry| entry.path())
.collect();
paths.sort();
for path in paths {
if path.is_dir() && !dirs_too {
continue;
}
let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
continue;
};
let Some(kind) = artifact_kind(name) else {
continue;
};
if kind == PruneArtifactKind::QuarantinedIndex
&& !fs::symlink_metadata(&path)
.map(|metadata| metadata.is_file())
.unwrap_or(false)
{
continue;
}
if let Some(filter) = original_filter
&& registered_artifact_original(name) != Some(filter)
{
continue;
}
artifacts.push(PruneArtifact {
bytes: recursive_size(&path),
age_days: age_days(&path, name),
path,
kind,
action: PruneAction::Kept,
});
}
}
pub fn enumerate_maintenance_artifacts(state_roots: &[PathBuf]) -> Vec<PruneArtifact> {
enumerate_maintenance_artifacts_filtered(state_roots, None)
}
pub fn enumerate_maintenance_artifacts_filtered(
state_roots: &[PathBuf],
realm_filter: Option<&str>,
) -> Vec<PruneArtifact> {
let realm_dir_name = realm_filter.map(meerkat_core::sanitize_realm_id);
let mut artifacts = Vec::new();
let mut seen_roots: Vec<PathBuf> = Vec::new();
for root in state_roots {
let canonical = fs::canonicalize(root).unwrap_or_else(|_| root.clone());
if seen_roots.contains(&canonical) {
continue;
}
seen_roots.push(canonical);
push_artifacts_in(root, true, realm_dir_name.as_deref(), &mut artifacts);
for realm in list_realm_dirs(root) {
if let Some(filter) = realm_filter
&& realm.realm_id != filter
{
continue;
}
for scan_dir in [
realm.dir.clone(),
realm.dir.join("memory"),
realm.dir.join("sessions_jsonl"),
realm.dir.join("mobs"),
] {
push_artifacts_in(&scan_dir, false, None, &mut artifacts);
}
}
}
artifacts
}
pub fn remove_maintenance_artifact(path: &Path) -> Result<(), StoreError> {
let name = path
.file_name()
.and_then(|name| name.to_str())
.unwrap_or_default();
if artifact_kind(name).is_none() {
return Err(StoreError::Internal(format!(
"refusing to remove '{}': not a registered maintenance artifact \
(*.pre-* / *.corrupt-*)",
path.display()
)));
}
restore_write_permissions_best_effort(path);
let metadata = fs::symlink_metadata(path)?;
if metadata.is_dir() {
fs::remove_dir_all(path)?;
} else {
fs::remove_file(path)?;
}
Ok(())
}
#[cfg(test)]
#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
mod tests {
#[test]
fn artifact_name_validation_rejects_lookalikes() {
for lookalike in [
"notes.pre-release",
"config.pre-prod",
".pre-0.8.3-1700000000",
"db.pre-0.8.3-notadigit",
"db.pre-083-1700000000",
"db.pre-0.8.3-1700000000.",
"db.pre-0.8.3-1700000000.bad purpose",
] {
assert!(!is_backup_artifact_name(lookalike), "{lookalike}");
}
for valid in [
"sessions.sqlite3.pre-0.8.3-1700000000",
"sessions.sqlite3.pre-0.8.3-1700000000.split-brain",
"realm-dir.pre-10.20.30-1.adopt_other",
] {
assert!(is_backup_artifact_name(valid), "{valid}");
}
for lookalike in ["notes.corrupt-ish", ".corrupt-123", "x.corrupt-12a"] {
assert!(!is_quarantine_artifact_name(lookalike), "{lookalike}");
}
assert!(is_quarantine_artifact_name(
"session_index.sqlite3.corrupt-1700000000"
));
}
use super::*;
use rusqlite::Connection;
fn create_db(path: &Path) {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).unwrap();
}
let conn = Connection::open(path).unwrap();
conn.execute_batch("CREATE TABLE t (x INTEGER)").unwrap();
}
fn write_manifest(state_root: &Path, realm_id: &str, backend: &str) -> PathBuf {
let dir = state_root.join(meerkat_core::sanitize_realm_id(realm_id));
fs::create_dir_all(&dir).unwrap();
fs::write(
dir.join(REALM_MANIFEST_FILE_NAME),
serde_json::to_vec(&serde_json::json!({
"realm_id": realm_id,
"backend": backend,
"origin": "explicit",
"created_at": "0",
}))
.unwrap(),
)
.unwrap();
dir
}
#[test]
fn fence_acquires_full_inventory_and_mobs_in_sorted_order() {
let temp = tempfile::tempdir().unwrap();
let realm = temp.path().join("realm");
create_db(&realm.join("sessions.sqlite3"));
create_db(&realm.join("workgraph.sqlite3"));
create_db(&realm.join("memory/memory.sqlite3"));
create_db(&realm.join("mobs/alpha.db"));
create_db(&realm.join("mobs/realm_profiles.db"));
fs::write(realm.join("notes.txt"), b"x").unwrap();
let fence = RealmMaintenanceFence::acquire(&realm, Duration::from_secs(1)).unwrap();
assert_eq!(fence.len(), REALM_SQLITE_FILES.len() + 2);
assert!(!fence.is_empty());
let mut expected: Vec<PathBuf> = REALM_SQLITE_FILES
.iter()
.map(|relative| realm.join(relative))
.collect();
expected.push(realm.join("mobs/alpha.db"));
expected.push(realm.join("mobs/realm_profiles.db"));
expected.sort();
assert_eq!(fence.fenced_databases(), expected.as_slice());
for database in fence.fenced_databases() {
assert!(meerkat_sqlite::fence_lock_path(database).is_file());
}
assert!(!realm.join("tasks.db").exists());
assert!(fence.admission_lock_path().ends_with("realm.mfence"));
assert!(fence.admission_lock_path().is_file());
}
#[test]
fn foreign_admission_holder_blocks_fence_acquisition() {
let temp = tempfile::tempdir().unwrap();
let realm = temp.path().join("realm");
create_db(&realm.join("sessions.sqlite3"));
let admission_lock = meerkat_sqlite::fence_lock_path(&realm_write_admission_target(&realm));
let foreign = fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(&admission_lock)
.unwrap();
foreign.try_lock().unwrap();
let error = RealmMaintenanceFence::acquire(&realm, Duration::from_millis(100))
.expect_err("foreign admission holder must fail acquisition");
assert!(
matches!(error, StoreError::MaintenanceFenceHeld { ref path }
if path.ends_with(REALM_WRITE_ADMISSION_STEM)),
"{error:?}"
);
drop(foreign);
RealmMaintenanceFence::acquire(&realm, Duration::from_secs(1)).unwrap();
}
#[test]
fn foreign_holder_fails_typed_and_releases_partial_acquisition() {
let temp = tempfile::tempdir().unwrap();
let realm = temp.path().join("realm");
create_db(&realm.join("sessions.sqlite3"));
create_db(&realm.join("workgraph.sqlite3"));
let foreign_lock = meerkat_sqlite::fence_lock_path(&realm.join("workgraph.sqlite3"));
let foreign = fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(&foreign_lock)
.unwrap();
foreign.try_lock().unwrap();
let error = RealmMaintenanceFence::acquire(&realm, Duration::from_millis(100))
.expect_err("foreign fence holder must fail acquisition");
assert!(
matches!(error, StoreError::MaintenanceFenceHeld { ref path }
if path.ends_with("workgraph.sqlite3")),
"{error:?}"
);
let reacquired =
meerkat_sqlite::ExclusiveFence::try_acquire(&realm.join("sessions.sqlite3")).unwrap();
assert!(reacquired.is_some(), "partial acquisition must be released");
drop(reacquired);
let admission =
meerkat_sqlite::ExclusiveFence::try_acquire(&realm_write_admission_target(&realm))
.unwrap();
assert!(admission.is_some(), "admission fence must be released");
drop(admission);
drop(foreign);
let fence = RealmMaintenanceFence::acquire(&realm, Duration::from_secs(1)).unwrap();
assert_eq!(fence.len(), REALM_SQLITE_FILES.len());
}
#[test]
fn empty_realm_still_fences_the_full_fixed_inventory() {
let temp = tempfile::tempdir().unwrap();
let fence = RealmMaintenanceFence::acquire(temp.path(), Duration::ZERO).unwrap();
assert_eq!(fence.len(), REALM_SQLITE_FILES.len());
assert!(!fence.is_empty());
for relative in REALM_SQLITE_FILES {
let lock = meerkat_sqlite::fence_lock_path(&temp.path().join(relative));
assert!(lock.is_file(), "{}", lock.display());
}
}
#[test]
fn backup_names_follow_the_registered_discipline() {
let name = backup_artifact_name("sessions.sqlite3", "split-brain");
assert!(
name.starts_with(&format!(
"sessions.sqlite3.pre-{}-",
env!("CARGO_PKG_VERSION")
)),
"{name}"
);
assert!(name.ends_with(".split-brain"), "{name}");
assert!(is_backup_artifact_name(&name));
let bare = backup_artifact_name("team", "");
assert!(is_backup_artifact_name(&bare));
assert!(!bare.ends_with('.'), "{bare}");
assert!(is_quarantine_artifact_name(
"session_index.sqlite3.corrupt-1"
));
assert!(!is_backup_artifact_name("sessions.sqlite3"));
}
#[test]
fn archive_renames_and_strips_write_permission() {
let temp = tempfile::tempdir().unwrap();
let dir = temp.path().join("team");
fs::create_dir_all(&dir).unwrap();
fs::write(dir.join("sessions.sqlite3"), b"data").unwrap();
let archive = archive_path_read_only(&dir, "split-brain").unwrap();
assert!(!dir.exists(), "original must be gone");
assert!(archive.is_dir());
assert!(archive.join("sessions.sqlite3").is_file());
let name = archive.file_name().unwrap().to_str().unwrap();
assert!(is_backup_artifact_name(name), "{name}");
let inner = fs::metadata(archive.join("sessions.sqlite3")).unwrap();
assert!(inner.permissions().readonly(), "archive must be read-only");
remove_maintenance_artifact(&archive).unwrap();
assert!(!archive.exists());
}
#[test]
fn archive_reported_returns_archive_path_and_no_warnings_on_success() {
let temp = tempfile::tempdir().unwrap();
let file = temp.path().join("sessions.sqlite3");
fs::write(&file, b"data").unwrap();
let archived = archive_path_read_only_reported(&file, "split-brain").unwrap();
assert!(archived.archive.is_file());
assert!(
archived.warnings.is_empty(),
"successful hardening + parent fsync must not warn: {:?}",
archived.warnings
);
let name = archived.archive.file_name().unwrap().to_str().unwrap();
assert!(is_backup_artifact_name(name), "{name}");
remove_maintenance_artifact(&archived.archive).unwrap();
}
#[cfg(unix)]
#[test]
fn archive_hardening_never_chmods_through_symlinks() {
use std::os::unix::fs::PermissionsExt;
let temp = tempfile::tempdir().unwrap();
let external = temp.path().join("external.txt");
fs::write(&external, b"do not chmod").unwrap();
let dir = temp.path().join("team");
fs::create_dir_all(&dir).unwrap();
std::os::unix::fs::symlink(&external, dir.join("link")).unwrap();
let archived = archive_path_read_only_reported(&dir, "split-brain").unwrap();
let mode = fs::metadata(&external).unwrap().permissions().mode();
assert!(
mode & 0o200 != 0,
"external symlink target must stay writable (mode {mode:o})"
);
remove_maintenance_artifact(&archived.archive).unwrap();
let mode = fs::metadata(&external).unwrap().permissions().mode();
assert!(mode & 0o200 != 0, "mode {mode:o}");
assert!(external.is_file());
}
#[test]
fn remove_refuses_unregistered_paths() {
let temp = tempfile::tempdir().unwrap();
let victim = temp.path().join("precious.txt");
fs::write(&victim, b"do not touch").unwrap();
let error = remove_maintenance_artifact(&victim).expect_err("must refuse");
assert!(matches!(error, StoreError::Internal(_)));
assert!(victim.is_file(), "unregistered path must survive");
}
#[test]
fn split_brain_divergence_classifies_sessions_and_files() {
let temp = tempfile::tempdir().unwrap();
let dir_a = temp.path().join("a/team");
let dir_b = temp.path().join("b/team");
for dir in [&dir_a, &dir_b] {
fs::create_dir_all(dir).unwrap();
}
let ddl = "CREATE TABLE sessions (
session_id TEXT PRIMARY KEY,
created_at_ms INTEGER NOT NULL,
updated_at_ms INTEGER NOT NULL,
message_count INTEGER NOT NULL,
total_tokens INTEGER NOT NULL,
metadata_json TEXT NOT NULL,
session_json BLOB NOT NULL
)";
let insert = |conn: &Connection, id: &str, body: &str| {
conn.execute(
"INSERT INTO sessions VALUES (?1, 0, 0, 0, 0, '{}', ?2)",
rusqlite::params![id, body.as_bytes()],
)
.unwrap();
};
let shared = "00000000-0000-4000-8000-000000000001";
let divergent = "00000000-0000-4000-8000-000000000002";
let only_a = "00000000-0000-4000-8000-000000000003";
{
let conn = Connection::open(dir_a.join("sessions.sqlite3")).unwrap();
conn.execute_batch(ddl).unwrap();
insert(&conn, shared, "same");
insert(&conn, divergent, "version-a");
insert(&conn, only_a, "solo");
}
{
let conn = Connection::open(dir_b.join("sessions.sqlite3")).unwrap();
conn.execute_batch(ddl).unwrap();
insert(&conn, shared, "same");
insert(&conn, divergent, "version-b");
}
create_db(&dir_b.join("workgraph.sqlite3"));
let report = compute_split_brain_report("team", &[dir_a.clone(), dir_b.clone()]);
assert!(report.errors.is_empty(), "{:?}", report.errors);
assert_eq!(report.sessions_equal, 1);
let status_of = |id: &str| {
report
.sessions
.iter()
.find(|entry| entry.session_id == id)
.map(|entry| entry.status.clone())
};
assert_eq!(status_of(divergent), Some(DivergenceStatus::Divergent));
assert_eq!(
status_of(only_a),
Some(DivergenceStatus::OnlyIn { location: dir_a })
);
assert_eq!(
status_of(shared),
None,
"equal sessions are counted, not listed"
);
let workgraph = report
.files
.iter()
.find(|entry| entry.file == "workgraph.sqlite3")
.expect("workgraph file entry");
assert_eq!(
workgraph.status,
DivergenceStatus::OnlyIn { location: dir_b }
);
let sessions_file = report
.files
.iter()
.find(|entry| entry.file == "sessions.sqlite3")
.expect("sessions file entry");
assert_eq!(sessions_file.status, DivergenceStatus::Divergent);
assert!(report.comparison_is_conclusive());
}
#[test]
fn split_brain_covers_jsonl_blobs_and_artifacts() {
let temp = tempfile::tempdir().unwrap();
let dir_a = temp.path().join("a/team");
let dir_b = temp.path().join("b/team");
for dir in [&dir_a, &dir_b] {
fs::create_dir_all(dir.join("sessions_jsonl")).unwrap();
fs::create_dir_all(dir.join("artifacts")).unwrap();
}
for dir in [&dir_a, &dir_b] {
fs::write(dir.join("sessions_jsonl/equal.jsonl"), b"{\"same\":1}").unwrap();
}
fs::write(dir_a.join("sessions_jsonl/split.jsonl"), b"version-a").unwrap();
fs::write(dir_b.join("sessions_jsonl/split.jsonl"), b"version-b").unwrap();
fs::create_dir_all(dir_a.join("blobs/ab")).unwrap();
fs::write(dir_a.join("blobs/ab/abcd.json"), b"blob-bytes").unwrap();
fs::write(dir_a.join("artifacts/r1.json"), b"{\"v\":1}").unwrap();
fs::write(dir_b.join("artifacts/r1.json"), b"{\"v\":2}").unwrap();
let report = compute_split_brain_report("team", &[dir_a.clone(), dir_b]);
assert!(report.errors.is_empty(), "{:?}", report.errors);
let status_of = |file: &str| {
report
.files
.iter()
.find(|entry| entry.file == file)
.map(|entry| entry.status.clone())
};
assert_eq!(
status_of("sessions_jsonl/equal.jsonl"),
Some(DivergenceStatus::Equal)
);
assert_eq!(
status_of("sessions_jsonl/split.jsonl"),
Some(DivergenceStatus::Divergent)
);
assert_eq!(
status_of("blobs/ab/abcd.json"),
Some(DivergenceStatus::OnlyIn { location: dir_a })
);
assert_eq!(
status_of("artifacts/r1.json"),
Some(DivergenceStatus::Divergent)
);
assert!(report.comparison_is_conclusive());
}
#[test]
fn split_brain_read_failure_poisons_the_session_comparison() {
let temp = tempfile::tempdir().unwrap();
let dir_a = temp.path().join("a/team");
let dir_b = temp.path().join("b/team");
for dir in [&dir_a, &dir_b] {
fs::create_dir_all(dir).unwrap();
}
fs::write(dir_a.join("sessions.sqlite3"), b"this is not sqlite").unwrap();
{
let conn = Connection::open(dir_b.join("sessions.sqlite3")).unwrap();
conn.execute_batch(
"CREATE TABLE sessions (
session_id TEXT PRIMARY KEY,
created_at_ms INTEGER NOT NULL,
updated_at_ms INTEGER NOT NULL,
message_count INTEGER NOT NULL,
total_tokens INTEGER NOT NULL,
metadata_json TEXT NOT NULL,
session_json BLOB NOT NULL
);
INSERT INTO sessions VALUES
('00000000-0000-4000-8000-000000000001', 0, 0, 0, 0, '{}', X'AA');",
)
.unwrap();
}
let report = compute_split_brain_report("team", &[dir_a, dir_b]);
assert!(!report.errors.is_empty());
assert_eq!(report.sessions_equal, 0);
assert_eq!(report.sessions.len(), 1);
assert_eq!(report.sessions[0].status, DivergenceStatus::Unknown);
assert!(!report.comparison_is_conclusive());
}
#[cfg(unix)]
#[test]
fn split_brain_never_follows_symlinks_and_marks_them_unknown() {
let temp = tempfile::tempdir().unwrap();
let outside = temp.path().join("outside.json");
fs::write(&outside, b"external bytes").unwrap();
let dir_a = temp.path().join("a/team");
let dir_b = temp.path().join("b/team");
for dir in [&dir_a, &dir_b] {
fs::create_dir_all(dir.join("blobs")).unwrap();
}
std::os::unix::fs::symlink(&outside, dir_a.join("blobs/link.json")).unwrap();
fs::write(dir_b.join("blobs/link.json"), b"external bytes").unwrap();
let report = compute_split_brain_report("team", &[dir_a, dir_b]);
assert!(!report.errors.is_empty());
let entry = report
.files
.iter()
.find(|entry| entry.file == "blobs/link.json")
.expect("symlinked entry");
assert_eq!(entry.status, DivergenceStatus::Unknown);
assert!(!report.comparison_is_conclusive());
}
#[cfg(unix)]
#[test]
fn split_brain_never_digests_through_symlinked_databases() {
let temp = tempfile::tempdir().unwrap();
let outside = temp.path().join("outside.sqlite3");
{
let conn = Connection::open(&outside).unwrap();
conn.execute_batch(
"CREATE TABLE sessions (
session_id TEXT PRIMARY KEY,
created_at_ms INTEGER NOT NULL,
updated_at_ms INTEGER NOT NULL,
message_count INTEGER NOT NULL,
total_tokens INTEGER NOT NULL,
metadata_json TEXT NOT NULL,
session_json BLOB NOT NULL
);
INSERT INTO sessions VALUES
('00000000-0000-4000-8000-000000000001', 0, 0, 0, 0, '{}', X'AA');",
)
.unwrap();
}
let dir_a = temp.path().join("a/team");
let dir_b = temp.path().join("b/team");
fs::create_dir_all(&dir_a).unwrap();
fs::create_dir_all(&dir_b).unwrap();
std::os::unix::fs::symlink(&outside, dir_a.join("sessions.sqlite3")).unwrap();
fs::copy(&outside, dir_b.join("sessions.sqlite3")).unwrap();
let inventory = enumerate_realm_sqlite_inventory(&dir_a);
assert!(inventory.files.is_empty());
assert_eq!(inventory.symlinks, vec![dir_a.join("sessions.sqlite3")]);
assert!(enumerate_realm_sqlite_files(&dir_a).is_empty());
let report = compute_split_brain_report("team", &[dir_a, dir_b]);
assert!(
report
.errors
.iter()
.any(|error| error.contains("refusing to follow symlink")),
"{:?}",
report.errors
);
let entry = report
.files
.iter()
.find(|entry| entry.file == "sessions.sqlite3")
.expect("sessions database entry");
assert_eq!(entry.status, DivergenceStatus::Unknown);
assert_eq!(report.sessions.len(), 1, "{:?}", report.sessions);
assert_eq!(report.sessions[0].status, DivergenceStatus::Unknown);
assert!(!report.comparison_is_conclusive());
}
#[test]
fn ledger_baseline_surfaces_read_failures_and_future_versions() {
let temp = tempfile::tempdir().unwrap();
let realm = temp.path().join("team");
fs::create_dir_all(realm.join("sessions_jsonl")).unwrap();
fs::write(realm.join("tasks.db"), b"this is not sqlite").unwrap();
{
let conn =
Connection::open(realm.join("sessions_jsonl/session_index.sqlite3")).unwrap();
conn.execute_batch(
"CREATE TABLE meerkat_schema (domain TEXT PRIMARY KEY, version INTEGER NOT NULL);",
)
.unwrap();
conn.execute(
"INSERT INTO meerkat_schema VALUES ('jsonl-index', ?1)",
[i64::MAX],
)
.unwrap();
}
create_db(&realm.join("runtime.sqlite3"));
let baseline = read_realm_ledger_baseline(&realm);
assert_eq!(baseline.errors.len(), 1, "{:?}", baseline.errors);
assert!(
baseline.errors[0].contains("tasks.db"),
"{:?}",
baseline.errors
);
assert!(
!baseline
.rows
.iter()
.any(|row| row.database.ends_with("tasks.db")),
"an unreadable database must contribute no ledger rows"
);
assert_eq!(baseline.future.len(), 1);
assert_eq!(baseline.future[0].domain, "jsonl-index");
assert_eq!(baseline.future[0].found, i64::MAX);
assert_eq!(
baseline.future[0].supported,
crate::index::JSONL_INDEX_DOMAIN.supported_version()
);
let runtime_row = baseline
.rows
.iter()
.find(|row| row.database.ends_with("runtime.sqlite3"))
.expect("runtime row");
assert_eq!(runtime_row.domain, "runtime-store");
assert_eq!(runtime_row.version, None);
}
#[test]
fn prune_enumeration_sees_only_registered_patterns() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("realms");
let realm_dir = write_manifest(&root, "artifacts", "sqlite");
fs::write(
realm_dir.join("sessions.sqlite3.pre-0.0.1-1700000000"),
b"backup",
)
.unwrap();
let jsonl = realm_dir.join("sessions_jsonl");
fs::create_dir_all(&jsonl).unwrap();
fs::write(jsonl.join("session_index.sqlite3.corrupt-42"), b"q").unwrap();
let archived = root.join("team.pre-0.0.1-1700000000.split-brain");
fs::create_dir_all(&archived).unwrap();
fs::write(archived.join("sessions.sqlite3"), b"old").unwrap();
fs::write(realm_dir.join("sessions.sqlite3"), b"live").unwrap();
fs::write(realm_dir.join("notes.txt"), b"keep me").unwrap();
let artifacts = enumerate_maintenance_artifacts(&[root.clone(), root]);
let mut names: Vec<String> = artifacts
.iter()
.map(|artifact| {
artifact
.path
.file_name()
.unwrap()
.to_string_lossy()
.into_owned()
})
.collect();
names.sort();
assert_eq!(
names,
vec![
"session_index.sqlite3.corrupt-42".to_string(),
"sessions.sqlite3.pre-0.0.1-1700000000".to_string(),
"team.pre-0.0.1-1700000000.split-brain".to_string(),
],
"duplicate roots must not double-count"
);
let archived_entry = artifacts
.iter()
.find(|artifact| artifact.path == archived)
.expect("archived dir entry");
assert_eq!(archived_entry.kind, PruneArtifactKind::BackupArtifact);
assert_eq!(archived_entry.bytes, 3);
}
#[test]
fn quarantine_artifacts_must_be_files_even_at_root_level() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("realms");
fs::create_dir_all(&root).unwrap();
fs::create_dir_all(root.join("data.corrupt-1700000000")).unwrap();
fs::write(root.join("index.sqlite3.corrupt-1700000000"), b"q").unwrap();
let artifacts = enumerate_maintenance_artifacts(&[root]);
let names: Vec<&str> = artifacts
.iter()
.filter_map(|artifact| artifact.path.file_name().and_then(|name| name.to_str()))
.collect();
assert_eq!(names, vec!["index.sqlite3.corrupt-1700000000"]);
}
#[test]
fn report_shapes_round_trip_through_json() {
let mut report = MigrateReport {
mode: MigrateMode::Apply,
..MigrateReport::default()
};
let mut realm = RealmMigrateReport::new("team", PathBuf::from("/roots/a/team"));
realm.backend = Some("sqlite".to_string());
realm.ledger.push(LedgerBaselineEntry {
database: PathBuf::from("/roots/a/team/sessions.sqlite3"),
domain: "session-store".to_string(),
before: None,
after: Some(1),
action: LedgerBaselineAction::Stamped,
});
realm.adoption = Some(CheckpointAdoptionOutcome {
failures: Vec::new(),
scanned: 3,
already_verified: 1,
adopted: 2,
legacy_pending: 0,
refused: vec![],
skipped: None,
});
report.realms.push(realm);
report.split_brain.push(SplitBrainReport {
realm: "team".to_string(),
locations: vec![
PathBuf::from("/roots/a/team"),
PathBuf::from("/roots/b/team"),
],
sessions_equal: 4,
sessions: vec![SessionDivergenceEntry {
session_id: "s".to_string(),
status: DivergenceStatus::Divergent,
}],
files: vec![],
resolution: SplitBrainResolution::Archived {
adopted: PathBuf::from("/roots/a/team"),
archived: vec![PathBuf::from("/roots/b/team.pre-0.8.3-1-split-brain")],
},
errors: vec![],
});
report.split_brain.push(SplitBrainReport {
realm: "solo".to_string(),
locations: vec![PathBuf::from("/roots/a/solo")],
sessions_equal: 0,
sessions: vec![SessionDivergenceEntry {
session_id: "s2".to_string(),
status: DivergenceStatus::Unknown,
}],
files: vec![],
resolution: SplitBrainResolution::ArchiveFailed {
adopted: PathBuf::from("/roots/a/solo"),
archived: vec![PathBuf::from("/roots/b/solo.pre-0.8.3-1.split-brain")],
reason: "rename failed".to_string(),
},
errors: vec!["sessions unreadable".to_string()],
});
let json = serde_json::to_string(&report).expect("serialize");
let parsed: MigrateReport = serde_json::from_str(&json).expect("deserialize");
assert!(matches!(parsed.mode, MigrateMode::Apply));
assert_eq!(parsed.realms.len(), 1);
assert_eq!(
parsed.realms[0].ledger[0].action,
LedgerBaselineAction::Stamped
);
assert!(matches!(
parsed.split_brain[0].resolution,
SplitBrainResolution::Archived { .. }
));
assert!(matches!(
&parsed.split_brain[1].resolution,
SplitBrainResolution::ArchiveFailed { archived, .. } if archived.len() == 1
));
assert!(!parsed.split_brain[1].comparison_is_conclusive());
assert!(parsed.split_brain[0].comparison_is_conclusive());
assert!(!parsed.has_errors());
let sparse: PruneReport = serde_json::from_str(r#"{"future_field":1}"#).expect("sparse");
assert!(matches!(sparse.mode, MigrateMode::DryRun));
assert!(sparse.artifacts.is_empty());
}
#[test]
fn census_counts_legacy_rows() {
let temp = tempfile::tempdir().unwrap();
let db = temp.path().join("sessions.sqlite3");
{
let conn = Connection::open(&db).unwrap();
conn.execute_batch(
"CREATE TABLE sessions (
session_id TEXT PRIMARY KEY,
created_at_ms INTEGER NOT NULL,
updated_at_ms INTEGER NOT NULL,
message_count INTEGER NOT NULL,
total_tokens INTEGER NOT NULL,
metadata_json TEXT NOT NULL,
session_json BLOB NOT NULL
)",
)
.unwrap();
let session = meerkat_core::Session::new();
conn.execute(
"INSERT INTO sessions VALUES (?1, 0, 0, 0, 0, ?2, ?3)",
rusqlite::params![
session.id().to_string(),
serde_json::to_string(session.metadata()).unwrap(),
serde_json::to_vec(&session).unwrap(),
],
)
.unwrap();
}
let census = census_legacy_sessions(&db).unwrap();
assert_eq!(census.legacy, 1);
assert_eq!(census.verified, 0);
assert_eq!(census.invalid, 0);
}
#[test]
fn read_domain_versions_reads_ledgers_read_only() {
let temp = tempfile::tempdir().unwrap();
let db = temp.path().join("db.sqlite3");
create_db(&db);
assert!(read_domain_versions(&db).unwrap().is_none());
{
let conn = Connection::open(&db).unwrap();
conn.execute_batch(
"CREATE TABLE meerkat_schema (domain TEXT PRIMARY KEY, version INTEGER NOT NULL);
INSERT INTO meerkat_schema VALUES ('session-store', 1);",
)
.unwrap();
}
assert_eq!(
read_domain_versions(&db).unwrap(),
Some(vec![("session-store".to_string(), 1)])
);
}
#[test]
fn list_realm_dirs_skips_archives_and_reads_manifests_leniently() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("realms");
write_manifest(&root, "alpha", "sqlite");
let corrupt = root.join("corrupt");
fs::create_dir_all(&corrupt).unwrap();
fs::write(corrupt.join(REALM_MANIFEST_FILE_NAME), b"not-json").unwrap();
let archived = root.join("beta.pre-0.0.1-1700000000");
fs::create_dir_all(&archived).unwrap();
fs::write(
archived.join(REALM_MANIFEST_FILE_NAME),
serde_json::to_vec(&serde_json::json!({
"realm_id": "beta", "backend": "sqlite",
"origin": "explicit", "created_at": "0",
}))
.unwrap(),
)
.unwrap();
fs::create_dir_all(root.join("not-a-realm")).unwrap();
let realms = list_realm_dirs(&root);
let ids: Vec<&str> = realms.iter().map(|realm| realm.realm_id.as_str()).collect();
assert_eq!(ids, vec!["alpha", "corrupt"]);
assert!(realms[0].manifest_readable);
assert_eq!(realms[0].backend.as_deref(), Some("sqlite"));
assert!(!realms[1].manifest_readable);
}
}