use std::collections::{HashMap, HashSet, VecDeque};
use std::io;
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime};
use khive_runtime::{KhiveRuntime, RuntimeError};
use khive_storage::types::{SqlStatement, SqlValue};
use super::ingest::{self, LineTailSource};
fn is_link_or_reparse(metadata: &std::fs::Metadata) -> bool {
if metadata.file_type().is_symlink() {
return true;
}
#[cfg(windows)]
{
use std::os::windows::fs::MetadataExt;
metadata.file_attributes()
& windows_sys::Win32::Storage::FileSystem::FILE_ATTRIBUTE_REPARSE_POINT
!= 0
}
#[cfg(not(windows))]
{
false
}
}
fn discovered_entry_is_link_or_reparse(
path: &Path,
file_type: &std::fs::FileType,
) -> io::Result<bool> {
if file_type.is_symlink() {
return Ok(true);
}
#[cfg(windows)]
{
Ok(is_link_or_reparse(&std::fs::symlink_metadata(path)?))
}
#[cfg(not(windows))]
{
let _ = path;
Ok(false)
}
}
#[derive(Clone, PartialEq, Eq)]
enum DiscoveredKind {
LineTail {
source: LineTailSource,
session_id: Option<String>,
},
ChatGptExport,
ClaudeAiExport,
}
const DIRECTORY_PROBES_PER_TICK: usize = 64;
const COLD_FILE_PROBES_PER_TICK: usize = 256;
const COLD_STALE_POPS_PER_TICK: usize = COLD_FILE_PROBES_PER_TICK * 4;
const DIRECTORY_FORCE_RESCAN_PROBES: u16 = 30;
const FILE_UNCHANGED_POLLS_BEFORE_COLD: u8 = 2;
const FILE_COLD_AGE: Duration = Duration::from_secs(5 * 60);
const FILE_UNCHANGED_POLLS_WITHOUT_MTIME: u8 = 30;
const FILE_ERROR_POLLS_BEFORE_COLD: u8 = 3;
const FILE_MISSING_PROBES_BEFORE_REMOVAL: u8 = 2;
const CURSOR_DELETE_RETRY_LIMIT: usize = 1024;
const DIRECTORY_REFRESH_FAILURES_BEFORE_WARN: u16 = 3;
pub struct MirrorConfig {
pub enabled: bool,
pub projects_dir: PathBuf,
pub codex_enabled: bool,
pub codex_sessions_dir: PathBuf,
pub chatgpt_enabled: bool,
pub chatgpt_exports_dir: PathBuf,
pub claude_ai_enabled: bool,
pub claude_ai_exports_dir: PathBuf,
pub poll_interval: Duration,
pub backfill: bool,
}
const DEFAULT_MIRROR_POLL_SECS: u64 = 2;
fn parse_mirror_poll_secs(raw: Option<&str>) -> u64 {
match raw {
None => DEFAULT_MIRROR_POLL_SECS,
Some(v) => match v.parse::<u64>() {
Ok(0) => {
tracing::warn!(
value = v,
default_secs = DEFAULT_MIRROR_POLL_SECS,
"KHIVE_MIRROR_POLL_SECS must be nonzero; using default"
);
DEFAULT_MIRROR_POLL_SECS
}
Ok(secs) => secs,
Err(_) => {
tracing::warn!(
value = v,
default_secs = DEFAULT_MIRROR_POLL_SECS,
"KHIVE_MIRROR_POLL_SECS is not numeric; using default"
);
DEFAULT_MIRROR_POLL_SECS
}
},
}
}
impl MirrorConfig {
pub fn from_env() -> Self {
let home = std::env::var("HOME").unwrap_or_else(|_| "/root".into());
let enabled = std::env::var("KHIVE_MIRROR_ENABLED")
.map(|v| matches!(v.to_lowercase().as_str(), "1" | "true" | "yes"))
.unwrap_or(false);
let projects_dir = std::env::var("KHIVE_MIRROR_PROJECTS_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from(&home).join(".claude").join("projects"));
let codex_enabled = std::env::var("KHIVE_MIRROR_CODEX_ENABLED")
.map(|v| matches!(v.to_lowercase().as_str(), "1" | "true" | "yes"))
.unwrap_or(false);
let codex_sessions_dir = std::env::var("KHIVE_MIRROR_CODEX_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from(&home).join(".codex").join("sessions"));
let chatgpt_enabled = std::env::var("KHIVE_MIRROR_CHATGPT_ENABLED")
.map(|v| matches!(v.to_lowercase().as_str(), "1" | "true" | "yes"))
.unwrap_or(false);
let chatgpt_exports_dir = std::env::var("KHIVE_MIRROR_CHATGPT_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from(&home).join(".chatgpt").join("exports"));
let claude_ai_enabled = std::env::var("KHIVE_MIRROR_CLAUDE_AI_ENABLED")
.map(|v| matches!(v.to_lowercase().as_str(), "1" | "true" | "yes"))
.unwrap_or(false);
let claude_ai_exports_dir = std::env::var("KHIVE_MIRROR_CLAUDE_AI_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from(&home).join(".claude").join("exports"));
let poll_raw = std::env::var("KHIVE_MIRROR_POLL_SECS").ok();
let poll_secs = parse_mirror_poll_secs(poll_raw.as_deref());
let backfill = std::env::var("KHIVE_MIRROR_BACKFILL")
.map(|v| !matches!(v.to_lowercase().as_str(), "0" | "false" | "no"))
.unwrap_or(true);
Self {
enabled,
projects_dir,
codex_enabled,
codex_sessions_dir,
chatgpt_enabled,
chatgpt_exports_dir,
claude_ai_enabled,
claude_ai_exports_dir,
poll_interval: Duration::from_secs(poll_secs),
backfill,
}
}
}
#[cfg(test)]
mod config_tests {
#[cfg(windows)]
use super::{ingest, DirectoryFingerprint};
use super::{parse_mirror_poll_secs, DirectoryKind, DiscoveredKind, DiscoveryIndex};
#[test]
fn poll_secs_zero_is_rejected_and_default_remains_two_seconds() {
assert_eq!(
parse_mirror_poll_secs(None),
2,
"missing value defaults to 2s"
);
assert_eq!(
parse_mirror_poll_secs(Some("abc")),
2,
"non-numeric value defaults to 2s"
);
assert_eq!(
parse_mirror_poll_secs(Some("0")),
2,
"explicit zero must be rejected back to the default, not accepted as a hot loop"
);
assert_eq!(
parse_mirror_poll_secs(Some("1")),
1,
"valid nonzero value is honored"
);
assert_eq!(
parse_mirror_poll_secs(Some("5")),
5,
"valid nonzero value is honored"
);
}
#[test]
fn claude_ai_scanner_accepts_only_nested_conversations_json() {
let temp = tempfile::TempDir::new().expect("temp dir");
let nested = temp.path().join("download").join("claude-export");
std::fs::create_dir_all(&nested).expect("nested export dir");
let export = nested.join("conversations.json");
std::fs::write(&export, "[]").expect("export fixture");
std::fs::write(nested.join("conversations-copy.json"), "[]").expect("non-matching fixture");
let mut discovery = DiscoveryIndex::default();
discovery.add_directory_tree(temp.path(), DirectoryKind::ClaudeAiExport, true);
assert_eq!(discovery.files.len(), 1);
assert!(matches!(
discovery
.files
.get(&export)
.and_then(|file| file.kinds.first()),
Some(&DiscoveredKind::ClaudeAiExport)
));
}
#[test]
fn overlapping_export_roots_retain_both_parser_candidates() {
let temp = tempfile::TempDir::new().expect("temp dir");
let export = temp.path().join("conversations.json");
std::fs::write(&export, "[]").expect("export fixture");
let mut discovery = DiscoveryIndex::default();
discovery.add_directory_tree(temp.path(), DirectoryKind::ChatGptExport, true);
discovery.add_directory_tree(temp.path(), DirectoryKind::ClaudeAiExport, true);
discovery.probe_directories();
let kinds = &discovery.files.get(&export).expect("tracked export").kinds;
assert_eq!(kinds.len(), 2);
assert!(matches!(
kinds.first(),
Some(&DiscoveredKind::ChatGptExport)
));
assert!(matches!(
kinds.get(1),
Some(&DiscoveredKind::ClaudeAiExport)
));
}
#[cfg(unix)]
#[test]
fn discovery_ignores_symlinked_transcript_outside_root() {
let root = tempfile::TempDir::new().expect("root");
let outside = tempfile::NamedTempFile::new().expect("outside transcript");
let link = root.path().join("linked.jsonl");
std::os::unix::fs::symlink(outside.path(), &link).expect("file symlink");
let mut discovery = DiscoveryIndex::default();
discovery.add_directory_tree(root.path(), DirectoryKind::ClaudeCodeProject, true);
assert!(!discovery.files.contains_key(&link));
}
#[cfg(unix)]
#[test]
fn configured_symlinked_export_root_is_refused_before_polling() {
let root = tempfile::TempDir::new().expect("root");
let target = tempfile::NamedTempFile::new().expect("export target");
std::fs::write(target.path(), "[]").expect("export fixture");
let link = root.path().join("conversations.json");
std::os::unix::fs::symlink(target.path(), &link).expect("export symlink");
let mut discovery = DiscoveryIndex::default();
discovery.add_export_root(
&link,
DirectoryKind::ChatGptExport,
DiscoveredKind::ChatGptExport,
);
assert!(discovery.files.is_empty());
assert!(discovery.directories.is_empty());
assert!(discovery.schedule_files().is_empty());
}
#[cfg(unix)]
#[test]
fn pinned_export_file_replaced_by_symlink_is_not_rescheduled() {
let root = tempfile::TempDir::new().expect("configured root");
let root_path = std::fs::canonicalize(root.path()).expect("physical configured root");
let outside = tempfile::TempDir::new().expect("outside root");
let export = root_path.join("conversations.json");
let outside_export = outside.path().join("conversations.json");
std::fs::write(&export, "[]").expect("configured export");
std::fs::write(&outside_export, "[{}]").expect("outside export");
let mut discovery = DiscoveryIndex::default();
discovery.add_directory_tree(&export, DirectoryKind::ChatGptExport, true);
assert!(discovery.files.get(&export).is_some_and(|file| file.pinned));
std::fs::remove_file(&export).expect("remove configured export");
std::os::unix::fs::symlink(&outside_export, &export).expect("replace with symlink");
discovery.add_directory_tree(&export, DirectoryKind::ChatGptExport, true);
discovery.probe_directories();
assert!(!discovery.files.contains_key(&export));
assert!(discovery
.directories
.get(&export)
.is_some_and(|dir| dir.pinned));
assert!(discovery.schedule_files().is_empty());
assert!(super::ingest::open_source_file_beneath(&root_path, &export, None).is_err());
}
#[cfg(unix)]
#[test]
fn initial_probe_refuses_a_linked_root_ancestor_in_a_writable_parent() {
use std::os::unix::fs::PermissionsExt;
let temp = tempfile::TempDir::new().expect("fixture directory");
let fixture = std::fs::canonicalize(temp.path()).expect("fixture anchor");
let inside = fixture.join("inside");
let root = inside.join("exports");
std::fs::create_dir_all(&root).expect("source root");
let export = root.join("conversations.json");
std::fs::write(&export, b"[]").expect("source fixture");
let (_, original_identity, _) =
super::probe_source_file(&root, &export).expect("ordinary initial probe");
let linked_ancestor = fixture.join("linked");
std::os::unix::fs::symlink(&inside, &linked_ancestor).expect("fixture ancestor link");
std::fs::set_permissions(&fixture, std::fs::Permissions::from_mode(0o777)).unwrap();
let linked_root = linked_ancestor.join("exports");
let linked_export = linked_root.join("conversations.json");
let mut discovery = DiscoveryIndex::default();
discovery.add_directory_tree(&linked_root, DirectoryKind::ChatGptExport, true);
assert!(discovery.files.contains_key(&linked_export));
assert!(
super::probe_source_file(&linked_root, &linked_export)
.is_err_and(|e| e.to_string().contains("refused: parent_permissions")),
"a link in an unsafe parent must refuse before the root identity is admitted"
);
let (_, reopened_identity, _) =
super::probe_source_file(&root, &export).expect("ordinary root remains usable");
assert_eq!(reopened_identity, original_identity);
assert_eq!(std::fs::read(&export).expect("unchanged source"), b"[]");
}
#[cfg(unix)]
#[test]
fn initial_probe_refuses_a_root_ancestor_substituted_after_discovery() {
use std::os::unix::fs::PermissionsExt;
use std::sync::{Arc, Barrier};
let temp = tempfile::TempDir::new().expect("fixture directory");
let fixture = std::fs::canonicalize(temp.path()).expect("fixture anchor");
let ancestor = fixture.join("configured");
let root = ancestor.join("exports");
let protected_ancestor = fixture.join("protected");
let protected_root = protected_ancestor.join("exports");
std::fs::create_dir_all(&root).expect("configured source root");
std::fs::create_dir_all(&protected_root).expect("protected fixture root");
let export = root.join("conversations.json");
let protected_export = protected_root.join("conversations.json");
std::fs::write(&export, b"[]").expect("configured fixture");
std::fs::write(&protected_export, b"[{}]").expect("protected fixture");
let protected_bytes = std::fs::read(&protected_export).expect("protected bytes before");
let protected_metadata =
std::fs::metadata(&protected_export).expect("protected metadata before");
let protected_identity = super::ingest::file_identity(
&std::fs::File::open(&protected_export).expect("protected fixture handle"),
)
.expect("protected identity before");
std::fs::set_permissions(&fixture, std::fs::Permissions::from_mode(0o777)).unwrap();
let mut discovery = DiscoveryIndex::default();
discovery.add_directory_tree(&root, DirectoryKind::ChatGptExport, true);
let scheduled = discovery.schedule_files();
assert_eq!(scheduled.len(), 1);
assert_eq!(scheduled[0].path, export);
let barrier = Arc::new(Barrier::new(2));
std::thread::scope(|scope| {
let swap_barrier = Arc::clone(&barrier);
let swap_ancestor = &ancestor;
let saved_ancestor = fixture.join("configured-original");
let swap_target = &protected_ancestor;
let swap = scope.spawn(move || -> std::io::Result<()> {
swap_barrier.wait();
std::fs::rename(swap_ancestor, saved_ancestor)?;
std::os::unix::fs::symlink(swap_target, swap_ancestor)
});
barrier.wait();
swap.join()
.expect("fixture substitution thread")
.expect("fixture substitution");
assert!(
super::probe_source_file(&root, &scheduled[0].path)
.is_err_and(|e| e.to_string().contains("refused: parent_permissions")),
"initial admission must refuse a root ancestor substituted after discovery"
);
});
let after = std::fs::metadata(&protected_export).expect("protected metadata after");
assert_eq!(
std::fs::read(&protected_export).expect("protected bytes after"),
protected_bytes
);
assert_eq!(after.len(), protected_metadata.len());
assert_eq!(
after.modified().expect("mtime after"),
protected_metadata.modified().expect("mtime before")
);
assert_eq!(
super::ingest::file_identity(
&std::fs::File::open(&protected_export).expect("protected fixture handle after"),
)
.expect("protected identity after"),
protected_identity
);
assert_eq!(
std::fs::read(fixture.join("configured-original/exports/conversations.json"))
.expect("configured source remains intact"),
b"[]"
);
}
#[cfg(windows)]
#[test]
fn mirror_windows_junction_is_neither_discovered_nor_opened() {
use std::os::windows::fs::MetadataExt;
use std::process::Command;
let temp = tempfile::TempDir::new().expect("tempdir");
let root = temp.path().join("root");
let outside = temp.path().join("outside");
std::fs::create_dir_all(&root).expect("root");
std::fs::create_dir_all(&outside).expect("outside");
let inside_file = root.join("conversations.json");
std::fs::write(&inside_file, "[]").expect("inside export");
std::fs::write(outside.join("conversations.json"), "[]").expect("outside export");
let junction = root.join("linked");
let output = Command::new("cmd")
.arg("/C")
.arg("mklink")
.arg("/J")
.arg(&junction)
.arg(&outside)
.output()
.expect("create junction");
assert!(
output.status.success(),
"junction creation failed: {output:?}"
);
let metadata = std::fs::symlink_metadata(&junction).expect("junction metadata");
assert_ne!(
metadata.file_attributes()
& windows_sys::Win32::Storage::FileSystem::FILE_ATTRIBUTE_REPARSE_POINT,
0
);
let mut discovery = DiscoveryIndex::default();
discovery.add_directory_tree(&root, DirectoryKind::ChatGptExport, true);
assert!(discovery.files.contains_key(&inside_file));
assert!(!discovery.directories.contains_key(&junction));
assert!(!discovery
.files
.contains_key(&junction.join("conversations.json")));
let fingerprint =
DirectoryFingerprint::from_metadata(&std::fs::metadata(&root).expect("root metadata"));
discovery
.refresh_directory(&root, &[DirectoryKind::ChatGptExport], fingerprint, true)
.expect("refresh root");
assert!(!discovery.directories.contains_key(&junction));
assert!(ingest::open_source_file_beneath(
&root,
&junction.join("conversations.json"),
None,
)
.is_err());
let mut configured = DiscoveryIndex::default();
configured.add_export_root(
&junction,
DirectoryKind::ChatGptExport,
DiscoveredKind::ChatGptExport,
);
assert!(configured.files.is_empty());
assert!(configured.directories.is_empty());
}
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum DirectoryKind {
ClaudeCodeRoot,
ClaudeCodeProject,
Codex,
ChatGptExport,
ClaudeAiExport,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct DirectoryFingerprint {
modified: Option<SystemTime>,
len: u64,
}
impl DirectoryFingerprint {
fn from_metadata(metadata: &std::fs::Metadata) -> Self {
Self {
modified: metadata.modified().ok(),
len: metadata.len(),
}
}
}
struct TrackedDirectory {
kinds: Vec<DirectoryKind>,
fingerprint: Option<DirectoryFingerprint>,
entries: HashSet<PathBuf>,
unchanged_probes: u16,
pinned: bool,
refresh_failures: u16,
}
struct TrackedFile {
kinds: Vec<DiscoveredKind>,
cold: bool,
unchanged_polls: u8,
pinned: bool,
consecutive_error_polls: u8,
missing_probes: u8,
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct CursorState {
byte_offset: u64,
file_identity: Option<String>,
}
impl CursorState {
fn reset_if_replaced(&mut self, identity: &str, file_len: u64, backfill: bool) -> bool {
let prior = self.file_identity.as_deref();
let identity_changed = prior.is_some_and(|old| old != identity);
let legacy_without_witness = prior.is_none();
let truncated = file_len < self.byte_offset;
if identity_changed || truncated || (legacy_without_witness && backfill) {
self.byte_offset = 0;
self.file_identity = Some(identity.to_string());
return true;
}
if legacy_without_witness {
self.file_identity = Some(identity.to_string());
}
false
}
}
async fn reconcile_cursor_identity(
runtime: &KhiveRuntime,
path: &Path,
cursor: &mut CursorState,
identity: &str,
file_len: u64,
backfill: bool,
) -> Result<bool, RuntimeError> {
let legacy_without_witness = cursor.file_identity.is_none();
let restarted = cursor.reset_if_replaced(identity, file_len, backfill);
if legacy_without_witness && !restarted {
if let Err(error) =
ingest::adopt_cursor_identity(runtime, path, cursor.byte_offset, identity).await
{
cursor.file_identity = None;
return Err(error);
}
}
Ok(restarted)
}
struct ScheduledFile {
path: PathBuf,
was_cold: bool,
}
#[derive(Default)]
struct CandidateDispatch {
stats: Option<ingest::MirrorStats>,
errors: Vec<RuntimeError>,
}
impl CandidateDispatch {
fn record(
&mut self,
result: Result<ingest::MirrorStats, RuntimeError>,
start_offset: u64,
) -> bool {
match result {
Ok(stats) if stats.new_offset > start_offset && stats.inserted > 0 => {
self.stats = Some(stats);
true
}
Ok(stats) if stats.new_offset > start_offset => {
let recorded_advancing = self
.stats
.as_ref()
.is_some_and(|recorded| recorded.new_offset > start_offset);
if !recorded_advancing {
self.stats = Some(stats);
}
false
}
Ok(stats) if stats.new_offset >= start_offset => {
if self.stats.is_none() {
self.stats = Some(stats);
}
false
}
Ok(_) => {
false
}
Err(error) => {
self.errors.push(error);
false
}
}
}
}
#[derive(Default)]
struct DirectoryProbeStats {
metadata_probes: usize,
walks: usize,
}
#[derive(Default)]
struct DiscoveryIndex {
directories: HashMap<PathBuf, TrackedDirectory>,
directory_queue: VecDeque<PathBuf>,
directory_enqueued: HashSet<PathBuf>,
files: HashMap<PathBuf, TrackedFile>,
hot_files: HashSet<PathBuf>,
priority_cold_queue: VecDeque<PathBuf>,
priority_cold_enqueued: HashSet<PathBuf>,
cold_queue: VecDeque<PathBuf>,
cold_enqueued: HashSet<PathBuf>,
removed_files: Vec<PathBuf>,
}
enum ClassifiedEntry {
Directory(DirectoryKind),
File(DiscoveredKind),
}
impl DiscoveryIndex {
fn from_config(config: &MirrorConfig) -> Self {
let mut discovery = Self::default();
if config.enabled {
discovery.add_directory_tree(&config.projects_dir, DirectoryKind::ClaudeCodeRoot, true);
}
if config.codex_enabled {
discovery.add_directory_tree(&config.codex_sessions_dir, DirectoryKind::Codex, true);
}
if config.chatgpt_enabled {
discovery.add_export_root(
&config.chatgpt_exports_dir,
DirectoryKind::ChatGptExport,
DiscoveredKind::ChatGptExport,
);
}
if config.claude_ai_enabled {
discovery.add_export_root(
&config.claude_ai_exports_dir,
DirectoryKind::ClaudeAiExport,
DiscoveredKind::ClaudeAiExport,
);
}
discovery
}
fn add_export_root(
&mut self,
path: &Path,
directory_kind: DirectoryKind,
file_kind: DiscoveredKind,
) {
if std::fs::symlink_metadata(path).is_ok_and(|metadata| is_link_or_reparse(&metadata)) {
tracing::warn!(
path = %path.display(),
"session mirror: refusing a linked configured export root"
);
return;
}
if path.is_file() {
if path.file_name().and_then(|name| name.to_str()) == Some("conversations.json") {
self.add_file(path.to_path_buf(), file_kind, true);
}
} else {
self.add_directory_tree(path, directory_kind, true);
}
}
fn add_directory_tree(&mut self, root: &Path, kind: DirectoryKind, pinned: bool) {
let mut pending = vec![(root.to_path_buf(), kind, pinned)];
while let Some((path, directory_kind, is_pinned)) = pending.pop() {
if let Some(directory) = self.directories.get_mut(&path) {
directory.pinned |= is_pinned;
if !directory.kinds.contains(&directory_kind) {
directory.kinds.push(directory_kind);
directory.fingerprint = None;
}
self.enqueue_directory(&path);
continue;
}
if self.files.contains_key(&path) {
self.remove_file(&path, false);
}
let metadata = match std::fs::symlink_metadata(&path) {
Ok(metadata) if is_link_or_reparse(&metadata) && is_pinned => None,
Ok(metadata) if is_link_or_reparse(&metadata) => continue,
Ok(metadata) if metadata.is_dir() => Some(metadata),
Ok(metadata)
if is_pinned
&& metadata.is_file()
&& path.file_name().and_then(|name| name.to_str())
== Some("conversations.json") =>
{
let file_kind = match directory_kind {
DirectoryKind::ChatGptExport => DiscoveredKind::ChatGptExport,
DirectoryKind::ClaudeAiExport => DiscoveredKind::ClaudeAiExport,
_ => continue,
};
self.add_file(path, file_kind, true);
continue;
}
Ok(_) if is_pinned => None,
Ok(_) => continue,
Err(_) if is_pinned => None,
Err(_) => continue,
};
let mut fingerprint = metadata.as_ref().map(DirectoryFingerprint::from_metadata);
let mut entries = HashSet::new();
let mut children = Vec::new();
if metadata.is_some() {
match std::fs::read_dir(&path) {
Ok(read_dir) => {
for entry in read_dir {
let Ok(entry) = entry else {
fingerprint = None;
continue;
};
let child_path = entry.path();
let Ok(file_type) = entry.file_type() else {
fingerprint = None;
continue;
};
let linked = match discovered_entry_is_link_or_reparse(
&child_path,
&file_type,
) {
Ok(linked) => linked,
Err(_) => {
fingerprint = None;
continue;
}
};
if linked {
continue;
}
let Some(classified) =
classify_entry(directory_kind, &child_path, file_type.is_dir())
else {
continue;
};
entries.insert(child_path.clone());
match classified {
ClassifiedEntry::Directory(child_kind) => {
children.push((child_path, child_kind, false));
}
ClassifiedEntry::File(file_kind) => {
self.add_file(child_path, file_kind, false);
}
}
}
}
Err(_) => fingerprint = None,
}
}
self.directories.insert(
path.clone(),
TrackedDirectory {
kinds: vec![directory_kind],
fingerprint,
entries,
unchanged_probes: 0,
pinned: is_pinned,
refresh_failures: 0,
},
);
self.enqueue_directory(&path);
pending.extend(children);
}
}
fn add_file(&mut self, path: PathBuf, kind: DiscoveredKind, pinned: bool) {
if let Some(file) = self.files.get_mut(&path) {
file.pinned |= pinned;
if !file.kinds.contains(&kind) {
file.kinds.push(kind);
}
return;
}
self.files.insert(
path.clone(),
TrackedFile {
kinds: vec![kind],
cold: false,
unchanged_polls: 0,
pinned,
consecutive_error_polls: 0,
missing_probes: 0,
},
);
self.hot_files.insert(path);
}
fn probe_directories(&mut self) -> DirectoryProbeStats {
let mut stats = DirectoryProbeStats::default();
let scheduled = self.directory_queue.len().min(DIRECTORY_PROBES_PER_TICK);
for _ in 0..scheduled {
let Some(path) = self.directory_queue.pop_front() else {
break;
};
self.directory_enqueued.remove(&path);
let Some(directory) = self.directories.get(&path) else {
continue;
};
let kinds = directory.kinds.clone();
let stored_fingerprint = directory.fingerprint;
let force_rescan = directory.unchanged_probes >= DIRECTORY_FORCE_RESCAN_PROBES;
stats.metadata_probes += 1;
match std::fs::symlink_metadata(&path) {
Ok(metadata) if is_link_or_reparse(&metadata) => {
self.remove_directory_tree(&path, true);
}
Ok(metadata) if metadata.is_dir() => {
if self.files.contains_key(&path) {
self.remove_file(&path, false);
}
let fingerprint = DirectoryFingerprint::from_metadata(&metadata);
if force_rescan || Some(fingerprint) != stored_fingerprint {
let changed = Some(fingerprint) != stored_fingerprint;
if let Err(error) =
self.refresh_directory(&path, &kinds, fingerprint, changed)
{
let failures = self.record_refresh_failure(&path);
if failures == DIRECTORY_REFRESH_FAILURES_BEFORE_WARN {
tracing::warn!(
path = %path.display(),
error = %error,
consecutive_failures = failures,
"session mirror: directory refresh keeps failing"
);
} else {
tracing::debug!(
path = %path.display(),
error = %error,
consecutive_failures = failures,
"session mirror: directory refresh failed"
);
}
} else {
self.clear_refresh_failures(&path);
stats.walks += 1;
}
} else if let Some(directory) = self.directories.get_mut(&path) {
directory.unchanged_probes = directory.unchanged_probes.saturating_add(1);
}
}
Ok(metadata)
if metadata.is_file()
&& path.file_name().and_then(|name| name.to_str())
== Some("conversations.json") =>
{
let pinned = self
.directories
.get(&path)
.is_some_and(|directory| directory.pinned);
self.remove_directory_tree(&path, true);
for kind in kinds {
let file_kind = match kind {
DirectoryKind::ChatGptExport => Some(DiscoveredKind::ChatGptExport),
DirectoryKind::ClaudeAiExport => Some(DiscoveredKind::ClaudeAiExport),
_ => None,
};
if let Some(file_kind) = file_kind {
self.add_file(path.clone(), file_kind, pinned);
}
}
}
Ok(_) => {
if self.files.contains_key(&path) {
self.remove_file(&path, false);
}
self.remove_directory_tree(&path, true);
}
Err(error) if error.kind() == io::ErrorKind::NotFound => {
self.remove_directory_tree(&path, true);
}
Err(error) => {
let failures = self.record_refresh_failure(&path);
if failures == DIRECTORY_REFRESH_FAILURES_BEFORE_WARN {
tracing::warn!(
path = %path.display(),
error = %error,
consecutive_failures = failures,
"session mirror: directory metadata probe keeps failing"
);
} else {
tracing::debug!(
path = %path.display(),
error = %error,
consecutive_failures = failures,
"session mirror: directory metadata probe failed"
);
}
}
}
if self.directories.contains_key(&path) {
self.enqueue_directory(&path);
}
}
stats
}
fn refresh_directory(
&mut self,
path: &Path,
kinds: &[DirectoryKind],
fingerprint: DirectoryFingerprint,
prioritize: bool,
) -> io::Result<()> {
let read_dir = std::fs::read_dir(path)?;
let mut entries = HashSet::new();
let mut directories = Vec::new();
let mut files = Vec::new();
let mut complete = true;
for entry in read_dir {
let entry = match entry {
Ok(entry) => entry,
Err(_) => {
complete = false;
continue;
}
};
let child_path = entry.path();
let file_type = match entry.file_type() {
Ok(file_type) => file_type,
Err(_) => {
complete = false;
continue;
}
};
let linked = match discovered_entry_is_link_or_reparse(&child_path, &file_type) {
Ok(linked) => linked,
Err(_) => {
complete = false;
continue;
}
};
if linked {
continue;
}
for kind in kinds {
let Some(classified) = classify_entry(*kind, &child_path, file_type.is_dir())
else {
continue;
};
entries.insert(child_path.clone());
match classified {
ClassifiedEntry::Directory(child_kind) => {
directories.push((child_path.clone(), child_kind));
}
ClassifiedEntry::File(file_kind) => {
files.push((child_path.clone(), file_kind));
}
}
}
}
let removed = if complete {
self.directories
.get(path)
.map(|directory| {
directory
.entries
.difference(&entries)
.cloned()
.collect::<Vec<_>>()
})
.unwrap_or_default()
} else {
Vec::new()
};
for removed_path in removed {
if self.directories.contains_key(&removed_path) {
self.remove_directory_tree(&removed_path, true);
} else {
self.remove_file(&removed_path, true);
}
}
for (file_path, file_kind) in files {
let already_tracked = self.files.contains_key(&file_path);
self.add_file(file_path.clone(), file_kind, false);
if prioritize && already_tracked {
self.prioritize_cold(&file_path);
}
}
for (directory_path, directory_kind) in directories {
self.add_directory_tree(&directory_path, directory_kind, false);
}
if let Some(directory) = self.directories.get_mut(path) {
if complete {
directory.fingerprint = Some(fingerprint);
directory.entries = entries;
} else {
directory.fingerprint = None;
}
directory.unchanged_probes = 0;
}
Ok(())
}
fn remove_directory_tree(&mut self, path: &Path, keep_pinned: bool) {
let Some(directory) = self.directories.get(path) else {
return;
};
let entries = directory.entries.iter().cloned().collect::<Vec<_>>();
let pinned = directory.pinned;
for entry in entries {
if self.directories.contains_key(&entry) {
self.remove_directory_tree(&entry, true);
} else {
self.remove_file(&entry, true);
}
}
if keep_pinned && pinned {
if let Some(directory) = self.directories.get_mut(path) {
directory.fingerprint = None;
directory.entries.clear();
directory.unchanged_probes = 0;
}
self.enqueue_directory(path);
} else {
self.directories.remove(path);
}
}
fn remove_file(&mut self, path: &Path, preserve_pinned: bool) {
if preserve_pinned && self.files.get(path).is_some_and(|file| file.pinned) {
return;
}
if self.files.remove(path).is_none() {
return;
}
self.hot_files.remove(path);
self.cold_enqueued.remove(path);
self.priority_cold_enqueued.remove(path);
self.removed_files.push(path.to_path_buf());
}
fn take_removed_files(&mut self) -> Vec<PathBuf> {
std::mem::take(&mut self.removed_files)
}
fn reactivate_file(&mut self, path: &Path) {
if let Some(file) = self.files.get_mut(path) {
file.cold = false;
file.unchanged_polls = 0;
file.missing_probes = 0;
file.consecutive_error_polls = 0;
self.hot_files.insert(path.to_path_buf());
}
}
fn schedule_files(&mut self) -> Vec<ScheduledFile> {
let mut scheduled = self
.hot_files
.iter()
.filter(|path| self.files.contains_key(*path))
.cloned()
.map(|path| ScheduledFile {
path,
was_cold: false,
})
.collect::<Vec<_>>();
let mut remaining_cold_probes = COLD_FILE_PROBES_PER_TICK;
let mut remaining_stale_pops = COLD_STALE_POPS_PER_TICK;
let mut selected_cold = HashSet::new();
let priority_candidates = self.priority_cold_queue.len();
for _ in 0..priority_candidates {
if remaining_stale_pops == 0 {
break;
}
let Some(path) = self.priority_cold_queue.pop_front() else {
break;
};
self.priority_cold_enqueued.remove(&path);
if self.files.get(&path).is_some_and(|file| file.cold)
&& selected_cold.insert(path.clone())
{
if remaining_cold_probes == 0 {
self.priority_cold_queue.push_front(path.clone());
self.priority_cold_enqueued.insert(path);
break;
}
scheduled.push(ScheduledFile {
path,
was_cold: true,
});
remaining_cold_probes -= 1;
} else {
remaining_stale_pops -= 1;
}
}
let cold_candidates = self.cold_queue.len();
for _ in 0..cold_candidates {
if remaining_stale_pops == 0 {
break;
}
let Some(path) = self.cold_queue.pop_front() else {
break;
};
self.cold_enqueued.remove(&path);
if self.files.get(&path).is_some_and(|file| file.cold)
&& selected_cold.insert(path.clone())
{
if remaining_cold_probes == 0 {
self.cold_queue.push_front(path.clone());
self.cold_enqueued.insert(path);
break;
}
scheduled.push(ScheduledFile {
path,
was_cold: true,
});
remaining_cold_probes -= 1;
} else {
remaining_stale_pops -= 1;
}
}
scheduled
}
fn record_unchanged(
&mut self,
path: &Path,
was_cold: bool,
modified: Option<SystemTime>,
now: SystemTime,
) {
let Some(file) = self.files.get_mut(path) else {
return;
};
file.missing_probes = 0;
if was_cold || file.cold {
file.cold = true;
self.enqueue_cold(path);
return;
}
file.unchanged_polls = file.unchanged_polls.saturating_add(1);
if should_mark_cold(file.unchanged_polls, modified, now) {
file.cold = true;
self.hot_files.remove(path);
self.enqueue_cold(path);
}
}
fn record_probe_error(&mut self, path: &Path, was_cold: bool, missing: bool) {
let pinned = self.files.get(path).is_some_and(|file| file.pinned);
if missing && !pinned {
let remove = if let Some(file) = self.files.get_mut(path) {
file.missing_probes = file.missing_probes.saturating_add(1);
file.missing_probes >= FILE_MISSING_PROBES_BEFORE_REMOVAL
} else {
false
};
if remove {
self.remove_file(path, false);
} else if was_cold {
self.enqueue_cold(path);
}
} else if was_cold {
self.enqueue_cold(path);
}
}
fn record_error_poll(&mut self, path: &Path) -> bool {
let Some(file) = self.files.get_mut(path) else {
return false;
};
file.consecutive_error_polls = file.consecutive_error_polls.saturating_add(1);
let crossed_threshold = file.consecutive_error_polls == FILE_ERROR_POLLS_BEFORE_COLD;
if file.consecutive_error_polls >= FILE_ERROR_POLLS_BEFORE_COLD && !file.cold {
file.cold = true;
file.unchanged_polls = 0;
self.hot_files.remove(path);
self.enqueue_cold(path);
}
crossed_threshold
}
fn clear_error_polls(&mut self, path: &Path) {
if let Some(file) = self.files.get_mut(path) {
file.consecutive_error_polls = 0;
}
}
fn enqueue_cold(&mut self, path: &Path) {
if self.files.contains_key(path) && self.cold_enqueued.insert(path.to_path_buf()) {
self.cold_queue.push_back(path.to_path_buf());
}
}
fn prioritize_cold(&mut self, path: &Path) {
if self.files.get(path).is_some_and(|file| file.cold)
&& self.priority_cold_enqueued.insert(path.to_path_buf())
{
self.priority_cold_queue.push_back(path.to_path_buf());
}
}
fn enqueue_directory(&mut self, path: &Path) {
if self.directories.contains_key(path) && self.directory_enqueued.insert(path.to_path_buf())
{
self.directory_queue.push_back(path.to_path_buf());
}
}
fn record_refresh_failure(&mut self, path: &Path) -> u16 {
self.directories
.get_mut(path)
.map(|directory| {
directory.refresh_failures = directory.refresh_failures.saturating_add(1);
directory.refresh_failures
})
.unwrap_or(0)
}
fn clear_refresh_failures(&mut self, path: &Path) {
if let Some(directory) = self.directories.get_mut(path) {
directory.refresh_failures = 0;
}
}
fn tracked_files(&self) -> usize {
self.files.len()
}
}
fn should_mark_cold(unchanged_polls: u8, modified: Option<SystemTime>, now: SystemTime) -> bool {
if unchanged_polls < FILE_UNCHANGED_POLLS_BEFORE_COLD {
return false;
}
match modified.and_then(|modified| now.duration_since(modified).ok()) {
Some(age) => age >= FILE_COLD_AGE,
None => unchanged_polls >= FILE_UNCHANGED_POLLS_WITHOUT_MTIME,
}
}
fn tally_dispatch_errors(
discovery: &mut DiscoveryIndex,
path: &Path,
advanced: bool,
had_errors: bool,
) -> bool {
if advanced {
discovery.clear_error_polls(path);
return false;
}
had_errors && discovery.record_error_poll(path)
}
async fn finalize_dispatch_stats(
runtime: &KhiveRuntime,
path: &Path,
offset: u64,
ended_by_inserting: bool,
stats: Option<ingest::MirrorStats>,
mut had_errors: bool,
) -> (Option<ingest::MirrorStats>, bool) {
let stats = match stats {
Some(stats) if !ended_by_inserting && stats.inserted == 0 && stats.new_offset > offset => {
let uncontested_skip = stats.skipped_oversized_bytes && stats.scanned == 0;
if had_errors && !uncontested_skip {
tracing::debug!(
path = %path.display(),
new_offset = stats.new_offset,
"session mirror: deferring empty-advance cursor commit because a \
candidate errored; the span will be re-read on a later pass"
);
None
} else {
if had_errors {
tracing::debug!(
path = %path.display(),
new_offset = stats.new_offset,
"session mirror: committing an oversized-line skip despite a \
sibling candidate error — the skip is source-independent"
);
}
match ingest::commit_empty_advance(runtime, path, &stats).await {
Ok(()) => Some(stats),
Err(error) => {
had_errors = true;
tracing::warn!(
path = %path.display(),
error = %error,
new_offset = stats.new_offset,
"session mirror: empty-advance cursor commit failed; \
offset held back for a bounded re-read"
);
None
}
}
}
}
other => other,
};
(stats, had_errors)
}
fn classify_entry(
directory_kind: DirectoryKind,
path: &Path,
is_directory: bool,
) -> Option<ClassifiedEntry> {
if is_directory {
return match directory_kind {
DirectoryKind::ClaudeCodeRoot => {
Some(ClassifiedEntry::Directory(DirectoryKind::ClaudeCodeProject))
}
DirectoryKind::ClaudeCodeProject => None,
DirectoryKind::Codex => Some(ClassifiedEntry::Directory(DirectoryKind::Codex)),
DirectoryKind::ChatGptExport => {
Some(ClassifiedEntry::Directory(DirectoryKind::ChatGptExport))
}
DirectoryKind::ClaudeAiExport => {
Some(ClassifiedEntry::Directory(DirectoryKind::ClaudeAiExport))
}
};
}
match directory_kind {
DirectoryKind::ClaudeCodeProject
if path.extension().and_then(|extension| extension.to_str()) == Some("jsonl") =>
{
Some(ClassifiedEntry::File(DiscoveredKind::LineTail {
source: LineTailSource::ClaudeCode,
session_id: None,
}))
}
DirectoryKind::Codex
if path.extension().and_then(|extension| extension.to_str()) == Some("jsonl") =>
{
extract_codex_session_id(path).map(|session_id| {
ClassifiedEntry::File(DiscoveredKind::LineTail {
source: LineTailSource::Codex,
session_id: Some(session_id),
})
})
}
DirectoryKind::ChatGptExport
if path.file_name().and_then(|name| name.to_str()) == Some("conversations.json") =>
{
Some(ClassifiedEntry::File(DiscoveredKind::ChatGptExport))
}
DirectoryKind::ClaudeAiExport
if path.file_name().and_then(|name| name.to_str()) == Some("conversations.json") =>
{
Some(ClassifiedEntry::File(DiscoveredKind::ClaudeAiExport))
}
_ => None,
}
}
fn trusted_root_for_file<'a>(
config: &'a MirrorConfig,
path: &Path,
kinds: &[DiscoveredKind],
) -> Option<&'a Path> {
kinds.iter().find_map(|kind| {
let configured = match kind {
DiscoveredKind::LineTail {
source: LineTailSource::ClaudeCode,
..
} => config.projects_dir.as_path(),
DiscoveredKind::LineTail {
source: LineTailSource::Codex,
..
} => config.codex_sessions_dir.as_path(),
DiscoveredKind::ChatGptExport => config.chatgpt_exports_dir.as_path(),
DiscoveredKind::ClaudeAiExport => config.claude_ai_exports_dir.as_path(),
};
let root = if configured == path {
configured.parent().unwrap_or(Path::new(""))
} else {
configured
};
path.strip_prefix(root)
.ok()
.filter(|relative| !relative.as_os_str().is_empty())
.map(|_| root)
})
}
fn probe_source_file(
root: &Path,
path: &Path,
) -> io::Result<(std::fs::Metadata, String, Vec<String>)> {
ingest::open_source_file_beneath(root, path, None).and_then(|(file, directory_identities)| {
let metadata = file.metadata()?;
let identity = ingest::file_identity(&file)?;
Ok((metadata, identity, directory_identities))
})
}
pub async fn run_mirror_service(runtime: KhiveRuntime, config: MirrorConfig) {
tracing::info!(
projects_dir = %config.projects_dir.display(),
codex_sessions_dir = %config.codex_sessions_dir.display(),
chatgpt_exports_dir = %config.chatgpt_exports_dir.display(),
claude_ai_exports_dir = %config.claude_ai_exports_dir.display(),
poll_interval_ms = config.poll_interval.as_millis(),
backfill = config.backfill,
cc_enabled = config.enabled,
codex_enabled = config.codex_enabled,
chatgpt_enabled = config.chatgpt_enabled,
claude_ai_enabled = config.claude_ai_enabled,
"session mirror service starting"
);
let mut cursors: HashMap<PathBuf, CursorState> = match load_cursors(&runtime).await {
Ok(map) => map,
Err(e) => {
tracing::warn!(error = %e, "session mirror: failed to load cursors (starting from empty)");
HashMap::new()
}
};
let mut discovery = DiscoveryIndex::from_config(&config);
let mut pending_cursor_deletes: VecDeque<PathBuf> = VecDeque::new();
loop {
let directory_work = discovery.probe_directories();
let removed_files = discovery.take_removed_files();
if !removed_files.is_empty() {
for removed in &removed_files {
cursors.remove(removed);
}
queue_cursor_deletes(&mut pending_cursor_deletes, &removed_files);
}
let blocked_cursor_restores = drain_pending_cursor_deletes(
&runtime,
&discovery,
&mut cursors,
&mut pending_cursor_deletes,
)
.await;
let scheduled = discovery
.schedule_files()
.into_iter()
.filter(|scheduled_file| {
!blocked_cursor_restores.contains(scheduled_file.path.as_path())
})
.collect::<Vec<_>>();
let total_tracked = discovery.tracked_files();
let mut files_mirrored: u64 = 0;
let mut rows_inserted: u64 = 0;
for scheduled_file in scheduled {
let Some(root) = discovery
.files
.get(&scheduled_file.path)
.and_then(|file| trusted_root_for_file(&config, &scheduled_file.path, &file.kinds))
else {
discovery.record_probe_error(&scheduled_file.path, scheduled_file.was_cold, false);
tracing::warn!(
path = %scheduled_file.path.display(),
"session mirror: scheduled file is outside its configured root"
);
continue;
};
let (metadata, observed_identity, directory_identities) =
match probe_source_file(root, &scheduled_file.path) {
Ok(probe) => probe,
Err(e) => {
let missing = e.kind() == io::ErrorKind::NotFound;
discovery.record_probe_error(
&scheduled_file.path,
scheduled_file.was_cold,
missing,
);
if !missing {
tracing::warn!(
path = %scheduled_file.path.display(),
error = %e,
"session mirror: stat failed"
);
} else {
tracing::debug!(
path = %scheduled_file.path.display(),
error = %e,
"session mirror: file missing during probe"
);
}
continue;
}
};
if !metadata.is_file() {
discovery.record_probe_error(&scheduled_file.path, scheduled_file.was_cold, false);
tracing::warn!(
path = %scheduled_file.path.display(),
"session mirror: refusing a non-regular file or symlink"
);
continue;
}
let file_len = metadata.len();
let modified = metadata.modified().ok();
let cursor = cursors
.entry(scheduled_file.path.clone())
.or_insert_with(|| CursorState {
byte_offset: if config.backfill { 0 } else { file_len },
file_identity: Some(observed_identity.clone()),
});
let previous_offset = cursor.byte_offset;
let restarted = match reconcile_cursor_identity(
&runtime,
&scheduled_file.path,
cursor,
&observed_identity,
file_len,
config.backfill,
)
.await
{
Ok(restarted) => restarted,
Err(error) => {
tracing::warn!(
path = %scheduled_file.path.display(),
error = %error,
"session mirror: could not persist legacy cursor identity"
);
continue;
}
};
if restarted {
tracing::info!(
path = %scheduled_file.path.display(),
previous_offset,
file_len,
"session mirror: file replacement or truncation; restarting at byte zero"
);
}
let offset = cursor.byte_offset;
if file_len <= offset {
discovery.record_unchanged(
&scheduled_file.path,
scheduled_file.was_cold,
modified,
SystemTime::now(),
);
continue;
}
discovery.reactivate_file(&scheduled_file.path);
let Some(kinds) = discovery
.files
.get(&scheduled_file.path)
.map(|file| file.kinds.clone())
else {
continue;
};
let mut candidate_dispatch = CandidateDispatch::default();
let mut ended_by_inserting = false;
let trusted_source = ingest::TrustedSource {
root,
directory_identities: &directory_identities,
};
for kind in kinds {
let result = match kind {
DiscoveredKind::LineTail { source, session_id } => {
ingest::mirror_file_deferred_checked(
&runtime,
&scheduled_file.path,
offset,
source,
session_id.as_deref(),
Some(&observed_identity),
Some(trusted_source),
)
.await
}
DiscoveredKind::ChatGptExport => {
ingest::mirror_chatgpt_export_file_checked(
&runtime,
&scheduled_file.path,
offset,
&observed_identity,
trusted_source,
)
.await
}
DiscoveredKind::ClaudeAiExport => {
ingest::mirror_claude_ai_export_file_checked(
&runtime,
&scheduled_file.path,
offset,
&observed_identity,
trusted_source,
)
.await
}
};
if candidate_dispatch.record(result, offset) {
ended_by_inserting = true;
break;
}
}
let CandidateDispatch { stats, errors } = candidate_dispatch;
let had_errors = !errors.is_empty();
for error in errors {
tracing::warn!(
path = %scheduled_file.path.display(),
error = %error,
"session mirror: per-file source candidate error"
);
}
let (stats, had_errors) = finalize_dispatch_stats(
&runtime,
&scheduled_file.path,
offset,
ended_by_inserting,
stats,
had_errors,
)
.await;
let advanced = stats
.as_ref()
.is_some_and(|stats| stats.new_offset > offset);
if tally_dispatch_errors(&mut discovery, &scheduled_file.path, advanced, had_errors) {
tracing::warn!(
path = %scheduled_file.path.display(),
consecutive_error_polls = FILE_ERROR_POLLS_BEFORE_COLD,
"session mirror: demoting persistently erroring file to cold"
);
}
if let Some(stats) = stats {
if stats.new_offset >= offset {
cursors.insert(
scheduled_file.path.clone(),
CursorState {
byte_offset: stats.new_offset,
file_identity: stats.file_identity,
},
);
}
if stats.inserted > 0 || stats.new_offset > offset {
files_mirrored += 1;
rows_inserted += stats.inserted;
tracing::debug!(
path = %scheduled_file.path.display(),
inserted = stats.inserted,
scanned = stats.scanned,
new_offset = stats.new_offset,
"session mirror: tailed file"
);
}
}
}
if files_mirrored > 0 || rows_inserted > 0 {
tracing::info!(
files_mirrored,
rows_inserted,
total_tracked,
directory_metadata_probes = directory_work.metadata_probes,
directory_walks = directory_work.walks,
"session mirror tick"
);
} else {
tracing::debug!(
total_tracked,
directory_metadata_probes = directory_work.metadata_probes,
directory_walks = directory_work.walks,
"session mirror: quiet tick"
);
}
tokio::time::sleep(config.poll_interval).await;
}
}
async fn load_cursors(
runtime: &KhiveRuntime,
) -> Result<HashMap<PathBuf, CursorState>, RuntimeError> {
let sql = runtime.sql();
let mut reader = sql
.reader()
.await
.map_err(|e| RuntimeError::Internal(format!("mirror: cursor reader: {e}")))?;
let rows = reader
.query_all(SqlStatement {
sql: "SELECT file_path, byte_offset, file_identity FROM session_mirror_cursor".into(),
params: vec![],
label: Some("mirror_load_cursors".into()),
})
.await;
match rows {
Err(e) => {
tracing::debug!(error = %e, "mirror: cursor table not yet available");
Ok(HashMap::new())
}
Ok(rows) => {
let mut map = HashMap::with_capacity(rows.len());
for row in rows {
let file_path = match row.get("file_path") {
Some(SqlValue::Text(s)) => PathBuf::from(s),
_ => continue,
};
let byte_offset = match row.get("byte_offset") {
Some(SqlValue::Integer(n)) => *n as u64,
_ => 0,
};
let file_identity = match row.get("file_identity") {
Some(SqlValue::Text(identity)) => Some(identity.clone()),
_ => None,
};
map.insert(
file_path,
CursorState {
byte_offset,
file_identity,
},
);
}
Ok(map)
}
}
}
async fn delete_cursors(
runtime: &KhiveRuntime,
paths: &[PathBuf],
) -> Result<Vec<PathBuf>, RuntimeError> {
let sql = runtime.sql();
let mut writer = sql
.writer()
.await
.map_err(|e| RuntimeError::Internal(format!("mirror: cursor writer: {e}")))?;
let mut failed = Vec::new();
for path in paths {
if let Err(error) = writer
.execute(SqlStatement {
sql: "DELETE FROM session_mirror_cursor WHERE file_path=?1".into(),
params: vec![SqlValue::Text(path.to_string_lossy().into_owned())],
label: Some("mirror_cursor_delete".into()),
})
.await
{
tracing::debug!(
path = %path.display(),
error = %error,
"session mirror: cursor delete failed for one path; continuing batch"
);
failed.push(path.clone());
}
}
Ok(failed)
}
fn queue_cursor_deletes(pending: &mut VecDeque<PathBuf>, removed: &[PathBuf]) {
for path in removed {
if pending.contains(path) {
continue;
}
if pending.len() >= CURSOR_DELETE_RETRY_LIMIT {
let dropped = pending.pop_front();
tracing::error!(
dropped = %dropped.map(|p| p.display().to_string()).unwrap_or_default(),
limit = CURSOR_DELETE_RETRY_LIMIT,
"session mirror: cursor-delete retry set full; evicting oldest entry — \
its stale cursor row can survive a daemon restart and skip bytes if \
the path is recreated"
);
}
pending.push_back(path.clone());
}
}
async fn drain_pending_cursor_deletes(
runtime: &KhiveRuntime,
discovery: &DiscoveryIndex,
cursors: &mut HashMap<PathBuf, CursorState>,
pending: &mut VecDeque<PathBuf>,
) -> HashSet<PathBuf> {
let mut blocked = HashSet::new();
if pending.is_empty() {
return blocked;
}
let mut cancelled = Vec::new();
pending.retain(|path| {
if discovery.files.contains_key(path) {
cancelled.push(path.clone());
false
} else {
true
}
});
let mut restore_failed = Vec::new();
for path in &cancelled {
if cursors.contains_key(path) {
continue;
}
match read_cursor_state(runtime, path).await {
Ok(Some(cursor)) => {
cursors.insert(path.clone(), cursor);
}
Ok(None) => {}
Err(error) => {
tracing::warn!(
path = %path.display(),
error = %error,
"session mirror: failed to restore cancelled cursor; retrying before scheduling"
);
restore_failed.push(path.clone());
blocked.insert(path.clone());
}
}
}
pending.extend(restore_failed);
if pending.is_empty() {
return blocked;
}
let paths: Vec<PathBuf> = pending
.iter()
.filter(|path| !blocked.contains(path.as_path()))
.cloned()
.collect();
if paths.is_empty() {
return blocked;
}
let blocked_pending = blocked.iter().cloned().collect::<VecDeque<_>>();
match delete_cursors(runtime, &paths).await {
Ok(failed) if failed.is_empty() => *pending = blocked_pending,
Ok(failed) => {
let mut remaining = blocked_pending;
remaining.extend(failed.iter().cloned());
tracing::warn!(
failed = failed.len(),
remaining = remaining.len(),
"session mirror: cursor cleanup partially failed; retrying failed paths next tick"
);
*pending = remaining;
}
Err(error) => {
tracing::warn!(
error = %error,
remaining = pending.len(),
"session mirror: cursor cleanup failed before any delete; retrying next tick"
);
}
}
blocked
}
async fn read_cursor_state(
runtime: &KhiveRuntime,
path: &Path,
) -> Result<Option<CursorState>, RuntimeError> {
let sql = runtime.sql();
let mut reader = sql.reader().await?;
let rows = reader
.query_all(SqlStatement {
sql: "SELECT byte_offset, file_identity FROM session_mirror_cursor WHERE file_path=?1"
.into(),
params: vec![SqlValue::Text(path.to_string_lossy().into_owned())],
label: Some("mirror_cursor_read".into()),
})
.await?;
Ok(rows.first().map(|row| CursorState {
byte_offset: match row.get("byte_offset") {
Some(SqlValue::Integer(offset)) => *offset as u64,
_ => 0,
},
file_identity: match row.get("file_identity") {
Some(SqlValue::Text(identity)) => Some(identity.clone()),
_ => None,
},
}))
}
fn extract_codex_session_id(path: &std::path::Path) -> Option<String> {
let stem = path.file_stem()?.to_str()?;
if !stem.starts_with("rollout-") {
return None;
}
let parts: Vec<&str> = stem.split('-').collect();
if parts.len() < 6 {
return None;
}
let candidate = parts[parts.len() - 5..].join("-");
match uuid::Uuid::parse_str(&candidate) {
Ok(_) => Some(candidate),
Err(_) => {
tracing::debug!(
path = %path.display(),
candidate,
"session mirror: codex filename did not yield a valid UUID — skipping"
);
None
}
}
}
#[cfg(test)]
mod discovery_tests {
use super::{
should_mark_cold, CandidateDispatch, DirectoryKind, DiscoveredKind, DiscoveryIndex,
TrackedDirectory, COLD_FILE_PROBES_PER_TICK, COLD_STALE_POPS_PER_TICK,
DIRECTORY_FORCE_RESCAN_PROBES, DIRECTORY_PROBES_PER_TICK,
DIRECTORY_REFRESH_FAILURES_BEFORE_WARN, FILE_COLD_AGE, FILE_ERROR_POLLS_BEFORE_COLD,
FILE_UNCHANGED_POLLS_BEFORE_COLD, FILE_UNCHANGED_POLLS_WITHOUT_MTIME,
};
use crate::mirror::ingest::MirrorStats;
use std::collections::HashSet;
use std::fs::OpenOptions;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
fn mark_cold(discovery: &mut DiscoveryIndex, path: &Path) {
let file = discovery.files.get_mut(path).expect("tracked file");
file.cold = true;
file.unchanged_polls = FILE_UNCHANGED_POLLS_BEFORE_COLD;
discovery.hot_files.remove(path);
discovery.enqueue_cold(path);
}
#[test]
fn cold_scheduler_has_a_fixed_per_tick_file_probe_ceiling() {
let mut discovery = DiscoveryIndex::default();
let count = COLD_FILE_PROBES_PER_TICK * 8;
for index in 0..count {
let path = PathBuf::from(format!("/cold/session-{index}.jsonl"));
discovery.add_file(path.clone(), DiscoveredKind::ChatGptExport, false);
mark_cold(&mut discovery, &path);
}
let scheduled = discovery.schedule_files();
assert_eq!(scheduled.len(), COLD_FILE_PROBES_PER_TICK);
assert!(scheduled.iter().all(|file| file.was_cold));
assert_eq!(discovery.tracked_files(), count);
}
#[test]
fn divergent_provider_limit_no_progress_does_not_block_next_candidate() {
let start_offset = 41;
let mut dispatch = CandidateDispatch::default();
assert!(!dispatch.record(
Ok(MirrorStats {
inserted: 0,
scanned: 0,
new_offset: start_offset,
..Default::default()
}),
start_offset,
));
assert!(dispatch.record(
Ok(MirrorStats {
inserted: 2,
scanned: 2,
new_offset: 97,
..Default::default()
}),
start_offset,
));
let stats = dispatch.stats.expect("progressing provider candidate");
assert_eq!(stats.new_offset, 97);
assert_eq!(stats.inserted, 2);
}
#[test]
fn directory_scheduler_has_a_fixed_per_tick_metadata_ceiling() {
let temp = tempfile::TempDir::new().expect("temp dir");
let mut discovery = DiscoveryIndex::default();
for index in 0..DIRECTORY_PROBES_PER_TICK * 4 {
discovery.add_directory_tree(
&temp.path().join(format!("missing-{index}")),
DirectoryKind::Codex,
true,
);
}
let stats = discovery.probe_directories();
assert_eq!(stats.metadata_probes, DIRECTORY_PROBES_PER_TICK);
assert_eq!(stats.walks, 0);
}
#[test]
fn parent_mtime_change_prioritizes_a_cold_growing_file_in_that_cycle() {
let temp = tempfile::TempDir::new().expect("temp dir");
let project = temp.path().join("project-slug");
std::fs::create_dir(&project).expect("project dir");
let transcript = project.join("session.jsonl");
std::fs::write(&transcript, "first\n").expect("transcript fixture");
let mut discovery = DiscoveryIndex::default();
discovery.add_directory_tree(temp.path(), DirectoryKind::ClaudeCodeRoot, true);
mark_cold(&mut discovery, &transcript);
OpenOptions::new()
.append(true)
.open(&transcript)
.expect("open transcript")
.write_all(b"second\n")
.expect("grow transcript");
discovery
.directories
.get_mut(&project)
.expect("tracked project directory")
.fingerprint = None;
let stats = discovery.probe_directories();
assert_eq!(stats.walks, 1);
assert!(!discovery.hot_files.contains(&transcript));
assert!(discovery
.schedule_files()
.iter()
.any(|file| file.path == transcript && file.was_cold));
}
#[test]
fn cold_round_robin_is_a_fallback_when_append_does_not_change_parent_mtime() {
let mut discovery = DiscoveryIndex::default();
let count = COLD_FILE_PROBES_PER_TICK * 3 + 1;
let target = PathBuf::from(format!("/cold/session-{}.jsonl", count - 1));
for index in 0..count {
let path = PathBuf::from(format!("/cold/session-{index}.jsonl"));
discovery.add_file(path.clone(), DiscoveredKind::ChatGptExport, false);
mark_cold(&mut discovery, &path);
}
let mut found_on_cycle = None;
for cycle in 1..=4 {
let scheduled = discovery.schedule_files();
if scheduled.iter().any(|file| file.path == target) {
found_on_cycle = Some(cycle);
break;
}
for file in scheduled {
discovery.record_unchanged(
&file.path,
file.was_cold,
Some(UNIX_EPOCH),
SystemTime::now(),
);
}
}
assert_eq!(found_on_cycle, Some(4));
}
#[test]
fn missing_configured_root_stays_tracked_until_it_appears() {
let temp = tempfile::TempDir::new().expect("temp dir");
let root = temp.path().join("codex-sessions");
let mut discovery = DiscoveryIndex::default();
discovery.add_directory_tree(&root, DirectoryKind::Codex, true);
discovery.probe_directories();
assert!(discovery.directories.contains_key(&root));
let day = root.join("2026/08/02");
std::fs::create_dir_all(&day).expect("date tree");
let transcript =
day.join("rollout-2026-08-02T12-00-00-019a731e-4a58-71b1-a71f-a8d2f9782113.jsonl");
std::fs::write(&transcript, "{}\n").expect("codex transcript");
discovery.probe_directories();
assert!(discovery.files.contains_key(&transcript));
assert!(discovery.hot_files.contains(&transcript));
}
#[cfg(unix)]
#[test]
fn tracked_directory_replaced_by_symlink_is_removed_on_next_refresh() {
let temp = tempfile::TempDir::new().expect("temp dir");
let tracked = temp.path().join("tracked");
let moved = temp.path().join("moved");
let outside = tempfile::TempDir::new().expect("outside dir");
std::fs::create_dir(&tracked).expect("tracked dir");
let transcript = tracked.join("session.jsonl");
std::fs::write(&transcript, "inside\n").expect("inside transcript");
std::fs::write(outside.path().join("session.jsonl"), "outside\n")
.expect("outside transcript");
let mut discovery = DiscoveryIndex::default();
discovery.add_directory_tree(&tracked, DirectoryKind::ClaudeCodeProject, false);
assert!(discovery.files.contains_key(&transcript));
assert!(discovery
.schedule_files()
.iter()
.any(|file| file.path == transcript));
std::fs::rename(&tracked, &moved).expect("move tracked dir");
std::os::unix::fs::symlink(outside.path(), &tracked).expect("replace with symlink");
discovery.probe_directories();
assert!(!discovery.directories.contains_key(&tracked));
assert!(!discovery.files.contains_key(&transcript));
assert!(!discovery
.schedule_files()
.iter()
.any(|file| file.path == transcript));
assert!(discovery.take_removed_files().contains(&transcript));
}
#[test]
fn nested_configured_root_keeps_identity_across_ancestor_remove_and_recreate() {
let temp = tempfile::TempDir::new().expect("temp dir");
let ancestor = temp.path().join("exports");
let nested = ancestor.join("claude-ai");
std::fs::create_dir_all(&nested).expect("nested export root");
let export = nested.join("conversations.json");
std::fs::write(&export, "[]").expect("export fixture");
let mut discovery = DiscoveryIndex::default();
discovery.add_directory_tree(&ancestor, DirectoryKind::ChatGptExport, true);
discovery.add_directory_tree(&nested, DirectoryKind::ClaudeAiExport, true);
std::fs::remove_dir_all(&nested).expect("remove nested configured root");
discovery
.directories
.get_mut(&ancestor)
.expect("tracked ancestor")
.fingerprint = None;
discovery.probe_directories();
let missing_root = discovery
.directories
.get(&nested)
.expect("configured nested root remains scheduled");
assert!(missing_root.pinned);
assert!(missing_root.kinds.contains(&DirectoryKind::ClaudeAiExport));
std::fs::create_dir_all(&nested).expect("recreate nested configured root");
std::fs::write(&export, "[]").expect("recreated export fixture");
discovery
.directories
.get_mut(&ancestor)
.expect("tracked ancestor")
.fingerprint = None;
discovery.probe_directories();
let kinds = &discovery
.files
.get(&export)
.expect("recreated export discovered")
.kinds;
assert!(kinds.contains(&DiscoveredKind::ClaudeAiExport));
}
#[test]
fn old_unchanged_files_become_cold_only_after_the_threshold() {
let now = SystemTime::now();
let old = now.checked_sub(FILE_COLD_AGE).expect("old timestamp");
assert!(!should_mark_cold(
FILE_UNCHANGED_POLLS_BEFORE_COLD - 1,
Some(old),
now
));
assert!(should_mark_cold(
FILE_UNCHANGED_POLLS_BEFORE_COLD,
Some(old),
now
));
assert!(!should_mark_cold(
FILE_UNCHANGED_POLLS_BEFORE_COLD,
Some(now),
now
));
}
#[test]
fn future_mtime_falls_back_to_unchanged_poll_threshold() {
let now = SystemTime::now();
let future = now
.checked_add(Duration::from_secs(3600))
.expect("future timestamp");
assert!(
!should_mark_cold(FILE_UNCHANGED_POLLS_WITHOUT_MTIME - 1, Some(future), now),
"below the no-mtime threshold an unusable mtime must not mark cold"
);
assert!(
should_mark_cold(FILE_UNCHANGED_POLLS_WITHOUT_MTIME, Some(future), now),
"a future-dated mtime must fall back to the unchanged-poll threshold"
);
}
#[test]
fn regressing_candidate_offset_is_never_recorded() {
let start_offset = 100;
let mut dispatch = CandidateDispatch::default();
assert!(!dispatch.record(
Ok(MirrorStats {
inserted: 4,
scanned: 4,
new_offset: start_offset - 60,
..Default::default()
}),
start_offset,
));
assert!(
dispatch.stats.is_none(),
"a regressing cursor must not be recorded"
);
assert!(dispatch.record(
Ok(MirrorStats {
inserted: 1,
scanned: 1,
new_offset: start_offset + 50,
..Default::default()
}),
start_offset,
));
assert_eq!(
dispatch
.stats
.expect("advancing candidate recorded")
.new_offset,
150
);
}
#[test]
fn remove_file_clears_scheduler_state_and_readd_starts_clean() {
let mut discovery = DiscoveryIndex::default();
let path = PathBuf::from("/cold/session.jsonl");
discovery.add_file(path.clone(), DiscoveredKind::ChatGptExport, false);
mark_cold(&mut discovery, &path);
assert!(discovery.cold_enqueued.contains(&path));
discovery.remove_file(&path, false);
assert!(!discovery.cold_enqueued.contains(&path));
assert!(!discovery.priority_cold_enqueued.contains(&path));
assert!(discovery
.schedule_files()
.iter()
.all(|file| file.path != path));
discovery.add_file(path.clone(), DiscoveredKind::ChatGptExport, false);
mark_cold(&mut discovery, &path);
assert!(discovery
.schedule_files()
.iter()
.any(|file| file.path == path && file.was_cold));
}
#[test]
fn remove_directory_tree_clears_directory_schedule_state() {
let temp = tempfile::TempDir::new().expect("temp dir");
let root = temp.path().join("projects");
std::fs::create_dir_all(&root).expect("root dir");
let mut discovery = DiscoveryIndex::default();
discovery.add_directory_tree(&root, DirectoryKind::ClaudeCodeRoot, false);
assert!(discovery.directory_enqueued.contains(&root));
discovery.remove_directory_tree(&root, false);
assert!(discovery.directory_enqueued.contains(&root));
discovery.add_directory_tree(&root, DirectoryKind::ClaudeCodeRoot, false);
let stats = discovery.probe_directories();
assert_eq!(stats.metadata_probes, 1);
}
#[test]
fn removed_files_are_reported_once_for_cursor_cleanup() {
let mut discovery = DiscoveryIndex::default();
let path = PathBuf::from("/exports/conversations.json");
discovery.add_file(path.clone(), DiscoveredKind::ChatGptExport, false);
discovery.remove_file(&path, false);
assert_eq!(discovery.take_removed_files(), vec![path.clone()]);
assert!(discovery.take_removed_files().is_empty());
discovery.add_file(path.clone(), DiscoveredKind::ChatGptExport, true);
discovery.remove_file(&path, true);
assert!(discovery.files.contains_key(&path));
assert!(discovery.take_removed_files().is_empty());
}
#[test]
fn pinned_missing_file_stays_tracked_and_hot_for_retry() {
let mut discovery = DiscoveryIndex::default();
let pinned = PathBuf::from("/exports/conversations.json");
discovery.add_file(pinned.clone(), DiscoveredKind::ChatGptExport, true);
discovery.record_probe_error(&pinned, false, true);
assert!(discovery.files.contains_key(&pinned));
assert!(discovery.hot_files.contains(&pinned));
assert!(discovery
.schedule_files()
.iter()
.any(|file| file.path == pinned && !file.was_cold));
}
#[test]
fn transient_not_found_gets_one_grace_probe_before_removal() {
let mut discovery = DiscoveryIndex::default();
let transient = PathBuf::from("/projects/gone.jsonl");
discovery.add_file(transient.clone(), DiscoveredKind::ChatGptExport, false);
discovery.record_probe_error(&transient, false, true);
assert!(discovery.files.contains_key(&transient));
assert!(discovery.hot_files.contains(&transient));
assert!(discovery.take_removed_files().is_empty());
discovery.record_unchanged(&transient, false, None, SystemTime::now());
discovery.record_probe_error(&transient, false, true);
assert!(discovery.files.contains_key(&transient));
discovery.record_probe_error(&transient, false, true);
assert!(!discovery.files.contains_key(&transient));
assert!(!discovery.hot_files.contains(&transient));
assert_eq!(discovery.take_removed_files(), vec![transient.clone()]);
}
#[test]
fn stale_cold_queue_residue_is_pop_bounded_and_hot_files_still_scheduled() {
let mut discovery = DiscoveryIndex::default();
let stale_count = COLD_STALE_POPS_PER_TICK * 5;
for index in 0..stale_count {
let path = PathBuf::from(format!("/stale/session-{index}.jsonl"));
discovery.add_file(path.clone(), DiscoveredKind::ChatGptExport, false);
mark_cold(&mut discovery, &path);
discovery.reactivate_file(&path);
}
let fresh_count = COLD_FILE_PROBES_PER_TICK * 3;
let first_fresh = PathBuf::from("/cold/session-0.jsonl");
for index in 0..fresh_count {
let path = PathBuf::from(format!("/cold/session-{index}.jsonl"));
discovery.add_file(path.clone(), DiscoveredKind::ChatGptExport, false);
mark_cold(&mut discovery, &path);
}
let hot = PathBuf::from("/hot/session.jsonl");
discovery.add_file(hot.clone(), DiscoveredKind::ChatGptExport, false);
let queue_before = discovery.cold_queue.len();
assert_eq!(queue_before, stale_count + fresh_count);
let mut scheduled = discovery.schedule_files();
assert!(
scheduled
.iter()
.any(|file| file.path == hot && !file.was_cold),
"hot scheduling is unaffected by the residue backlog"
);
let drained = queue_before - discovery.cold_queue.len();
assert!(drained <= COLD_STALE_POPS_PER_TICK + COLD_FILE_PROBES_PER_TICK);
assert!(
discovery.cold_queue.len() > fresh_count / 2,
"most of the backlog survives one tick"
);
let mut ticks = 1;
while !scheduled.iter().any(|file| file.path == first_fresh) {
scheduled = discovery.schedule_files();
ticks += 1;
assert!(
ticks <= (stale_count / COLD_STALE_POPS_PER_TICK) + 2,
"stale residue must drain at the bounded per-tick rate"
);
}
assert!(
scheduled.iter().filter(|file| file.was_cold).count() <= COLD_FILE_PROBES_PER_TICK,
"the productive metadata-probe ceiling holds on the catch-up tick"
);
}
#[test]
fn persistently_erroring_hot_file_is_demoted_to_cold_and_recovers() {
let mut discovery = DiscoveryIndex::default();
let path = PathBuf::from("/broken/session.jsonl");
discovery.add_file(path.clone(), DiscoveredKind::ChatGptExport, false);
assert!(discovery.hot_files.contains(&path));
for _tick in 1..FILE_ERROR_POLLS_BEFORE_COLD {
assert!(!discovery.record_error_poll(&path));
assert!(discovery.hot_files.contains(&path));
assert!(!discovery.files[&path].cold);
}
assert!(discovery.record_error_poll(&path));
assert!(!discovery.hot_files.contains(&path));
assert!(discovery.files[&path].cold);
assert!(discovery.cold_enqueued.contains(&path));
assert!(!discovery.record_error_poll(&path));
discovery.reactivate_file(&path);
assert!(discovery.hot_files.contains(&path));
assert!(!discovery.files[&path].cold);
for _ in 1..FILE_ERROR_POLLS_BEFORE_COLD {
assert!(!discovery.record_error_poll(&path));
}
assert!(discovery.record_error_poll(&path));
}
#[test]
fn empty_advance_candidate_does_not_mask_inserting_provider() {
let start_offset = 10;
let mut dispatch = CandidateDispatch::default();
assert!(!dispatch.record(
Ok(MirrorStats {
inserted: 0,
scanned: 0,
new_offset: start_offset + 40,
..Default::default()
}),
start_offset,
));
assert!(dispatch.record(
Ok(MirrorStats {
inserted: 3,
scanned: 3,
new_offset: start_offset + 40,
..Default::default()
}),
start_offset,
));
let stats = dispatch.stats.expect("inserting provider recorded");
assert_eq!(stats.inserted, 3);
assert_eq!(stats.new_offset, start_offset + 40);
}
#[test]
fn later_empty_advance_replaces_earlier_no_progress_record() {
let start_offset = 100;
let mut dispatch = CandidateDispatch::default();
assert!(!dispatch.record(
Ok(MirrorStats {
inserted: 0,
scanned: 0,
new_offset: start_offset,
..Default::default()
}),
start_offset,
));
assert!(!dispatch.record(
Ok(MirrorStats {
inserted: 0,
scanned: 0,
new_offset: start_offset + 64,
..Default::default()
}),
start_offset,
));
let stats = dispatch.stats.expect("empty advance recorded");
assert_eq!(
stats.new_offset,
start_offset + 64,
"the advancing result wins over the earlier no-progress record"
);
assert_eq!(stats.inserted, 0);
}
#[test]
fn first_advancing_record_is_kept_against_later_empty_advance() {
let start_offset = 50;
let mut dispatch = CandidateDispatch::default();
assert!(!dispatch.record(
Ok(MirrorStats {
inserted: 0,
scanned: 0,
new_offset: start_offset + 30,
..Default::default()
}),
start_offset,
));
assert!(!dispatch.record(
Ok(MirrorStats {
inserted: 0,
scanned: 0,
new_offset: start_offset + 90,
..Default::default()
}),
start_offset,
));
assert_eq!(
dispatch
.stats
.expect("first advancing record kept")
.new_offset,
start_offset + 30
);
}
#[test]
fn mixed_no_progress_and_error_polls_count_toward_demotion() {
let mut discovery = DiscoveryIndex::default();
let path = PathBuf::from("/projects/mixed.jsonl");
discovery.add_file(path.clone(), DiscoveredKind::ChatGptExport, false);
for _ in 1..FILE_ERROR_POLLS_BEFORE_COLD {
assert!(
!super::tally_dispatch_errors(&mut discovery, &path, false, true),
"below the threshold no demotion"
);
assert!(!discovery.files[&path].cold);
}
assert!(
super::tally_dispatch_errors(&mut discovery, &path, false, true),
"the crossing tick reports the one-shot demotion warn"
);
assert!(
discovery.files[&path].cold,
"the mixed no-progress+error file is demoted to cold"
);
super::tally_dispatch_errors(&mut discovery, &path, true, false);
assert_eq!(discovery.files[&path].consecutive_error_polls, 0);
}
#[test]
fn refresh_failure_counter_crosses_the_warn_threshold_once_per_episode() {
let mut discovery = DiscoveryIndex::default();
let path = PathBuf::from("/projects/wedged");
discovery.directories.insert(
path.clone(),
TrackedDirectory {
kinds: vec![DirectoryKind::ClaudeCodeRoot],
fingerprint: None,
entries: HashSet::new(),
unchanged_probes: 0,
pinned: false,
refresh_failures: 0,
},
);
for _ in 1..DIRECTORY_REFRESH_FAILURES_BEFORE_WARN {
assert_ne!(
discovery.record_refresh_failure(&path),
DIRECTORY_REFRESH_FAILURES_BEFORE_WARN,
"below the threshold the log stays at debug"
);
}
assert_eq!(
discovery.record_refresh_failure(&path),
DIRECTORY_REFRESH_FAILURES_BEFORE_WARN,
"exactly one escalation per episode"
);
assert_ne!(
discovery.record_refresh_failure(&path),
DIRECTORY_REFRESH_FAILURES_BEFORE_WARN,
"subsequent failures in the same episode do not re-escalate"
);
discovery.clear_refresh_failures(&path);
assert_eq!(
discovery.directories[&path].refresh_failures, 0,
"a successful walk resets the episode"
);
}
#[test]
fn export_directory_becoming_a_file_keeps_pinned_identity_for_reversion() {
let temp = tempfile::TempDir::new().expect("temp dir");
let root = temp.path().join("conversations.json");
std::fs::create_dir(&root).expect("root directory");
let mut discovery = DiscoveryIndex::default();
discovery.add_directory_tree(&root, DirectoryKind::ChatGptExport, true);
std::fs::remove_dir(&root).expect("remove root directory");
std::fs::write(&root, "[]").expect("export file fixture");
discovery.probe_directories();
let file = discovery.files.get(&root).expect("export file tracked");
assert!(file.pinned);
assert!(file.kinds.contains(&DiscoveredKind::ChatGptExport));
let directory = discovery
.directories
.get(&root)
.expect("directory identity retained while the path is a file");
assert!(directory.pinned);
assert!(directory.kinds.contains(&DirectoryKind::ChatGptExport));
std::fs::remove_file(&root).expect("remove export file");
std::fs::create_dir(&root).expect("recreate root directory");
std::fs::write(root.join("conversations.json"), "[]").expect("nested export fixture");
discovery.probe_directories();
let directory = discovery
.directories
.get(&root)
.expect("configured root still scheduled after reversion");
assert!(directory.pinned);
assert!(directory.kinds.contains(&DirectoryKind::ChatGptExport));
assert!(
!discovery.files.contains_key(&root),
"the stale file record from the file phase must not shadow the directory"
);
assert!(discovery
.files
.contains_key(&root.join("conversations.json")));
}
#[test]
fn pinned_file_placeholder_at_startup_reverts_to_a_directory() {
let temp = tempfile::TempDir::new().expect("temp dir");
let root = temp.path().join("codex-sessions");
std::fs::write(&root, "unexpected regular file").expect("startup file fixture");
let mut discovery = DiscoveryIndex::default();
discovery.add_directory_tree(&root, DirectoryKind::Codex, true);
let placeholder = discovery
.directories
.get(&root)
.expect("pinned file gets a directory placeholder");
assert!(placeholder.pinned);
assert!(placeholder.kinds.contains(&DirectoryKind::Codex));
assert!(!discovery.files.contains_key(&root));
std::fs::remove_file(&root).expect("remove startup file");
std::fs::create_dir(&root).expect("recreate configured root as directory");
let day = root.join("2026/08/05");
std::fs::create_dir_all(&day).expect("date directory");
let transcript =
day.join("rollout-2026-08-05T12-00-00-019a731e-4a58-71b1-a71f-a8d2f9782113.jsonl");
std::fs::write(&transcript, "{}\n").expect("reversion transcript");
discovery.probe_directories();
let directory = discovery
.directories
.get(&root)
.expect("placeholder remains scheduled after reversion");
assert!(directory.pinned);
assert!(discovery.files.contains_key(&transcript));
}
#[test]
fn force_rescan_without_fingerprint_change_does_not_reprioritize_cold_files() {
let temp = tempfile::TempDir::new().expect("temp dir");
let project = temp.path().join("project-slug");
std::fs::create_dir(&project).expect("project dir");
let transcript = project.join("session.jsonl");
std::fs::write(&transcript, "first\n").expect("transcript fixture");
let mut discovery = DiscoveryIndex::default();
discovery.add_directory_tree(temp.path(), DirectoryKind::ClaudeCodeRoot, true);
mark_cold(&mut discovery, &transcript);
discovery.probe_directories();
discovery
.directories
.get_mut(&project)
.expect("tracked project directory")
.unchanged_probes = DIRECTORY_FORCE_RESCAN_PROBES;
let stats = discovery.probe_directories();
assert_eq!(stats.walks, 1);
assert!(
discovery.priority_cold_queue.is_empty(),
"a forced rescan of an unchanged directory carries no change signal"
);
assert!(discovery.cold_enqueued.contains(&transcript));
}
}
#[cfg(test)]
mod cursor_retry_tests {
use super::{
delete_cursors, drain_pending_cursor_deletes, finalize_dispatch_stats, load_cursors,
queue_cursor_deletes, reconcile_cursor_identity, tally_dispatch_errors, CandidateDispatch,
CursorState, DiscoveredKind, DiscoveryIndex, CURSOR_DELETE_RETRY_LIMIT,
FILE_ERROR_POLLS_BEFORE_COLD,
};
use crate::mirror::ingest::{file_identity, mirror_file, LineTailSource, MirrorStats};
use crate::vocab::{SESSION_SCHEMA_COLUMN_ADDITIONS, SESSION_SCHEMA_PLAN_STMTS};
use khive_runtime::{
AllowAllGate, BackendId, KhiveRuntime, Namespace, RuntimeConfig, RuntimeError,
};
use khive_storage::types::{SqlStatement, SqlValue};
use std::collections::HashMap;
use std::collections::VecDeque;
use std::io::Write;
use std::path::PathBuf;
use std::sync::Arc;
use tempfile::{NamedTempFile, TempDir};
fn runtime_without_schema() -> (KhiveRuntime, TempDir) {
let dir = TempDir::new().expect("tempdir");
let db_path = dir.path().join("test.db");
let rt = KhiveRuntime::new(RuntimeConfig {
wal_ceiling_bytes: 0,
wal_ceiling_configured_bytes: 0,
wal_ceiling_source: Default::default(),
wal_ceiling_env_raw: None,
web: Default::default(),
telemetry: Default::default(),
mounts: Vec::new(),
brain: Default::default(),
git_write: Default::default(),
display_timezone: khive_runtime::config::resolve_default_display_timezone(),
events_split: None,
db_path: Some(db_path),
blob_hydration_bytes: khive_runtime::DEFAULT_BLOB_HYDRATION_BYTES,
default_namespace: Namespace::local(),
embedding_model: None,
additional_embedding_models: vec![],
gate: Arc::new(AllowAllGate),
packs: vec!["kg".to_string()],
backend_id: BackendId::main(),
brain_profile: None,
visible_namespaces: vec![],
allowed_outbound_namespaces: vec![],
actor_id: None,
exec: Default::default(),
..khive_runtime::RuntimeConfig::no_embeddings()
})
.expect("file-backed runtime");
(rt, dir)
}
async fn apply_session_schema(rt: &KhiveRuntime) {
let sql = rt.sql();
let mut writer = sql.writer().await.expect("writer");
for stmt in &SESSION_SCHEMA_PLAN_STMTS {
writer
.execute_script(stmt.to_string())
.await
.expect("schema stmt");
}
}
async fn insert_cursor_row(rt: &KhiveRuntime, path: &std::path::Path, offset: i64) {
let mut writer = rt.sql().writer().await.expect("writer");
writer
.execute(SqlStatement {
sql: "INSERT INTO session_mirror_cursor(file_path, session_id, byte_offset, updated_at) \
VALUES(?1, NULL, ?2, 0)"
.into(),
params: vec![
SqlValue::Text(path.to_string_lossy().into_owned()),
SqlValue::Integer(offset),
],
label: None,
})
.await
.expect("cursor insert");
}
async fn cursor_row_exists(rt: &KhiveRuntime, path: &std::path::Path) -> bool {
let mut reader = rt.sql().reader().await.expect("reader");
let rows = reader
.query_all(SqlStatement {
sql: "SELECT byte_offset FROM session_mirror_cursor WHERE file_path = ?1".into(),
params: vec![SqlValue::Text(path.to_string_lossy().into_owned())],
label: None,
})
.await
.expect("cursor query");
!rows.is_empty()
}
async fn cursor_row_offset(rt: &KhiveRuntime, path: &std::path::Path) -> Option<i64> {
let mut reader = rt.sql().reader().await.expect("reader");
let rows = reader
.query_all(SqlStatement {
sql: "SELECT byte_offset FROM session_mirror_cursor WHERE file_path = ?1".into(),
params: vec![SqlValue::Text(path.to_string_lossy().into_owned())],
label: None,
})
.await
.expect("cursor query");
rows.first().and_then(|row| match row.get("byte_offset") {
Some(SqlValue::Integer(n)) => Some(*n),
_ => None,
})
}
#[test]
fn cursor_preserves_unchanged_and_append_offsets() {
let mut cursor = CursorState {
byte_offset: 100,
file_identity: Some("old-file".into()),
};
assert!(!cursor.reset_if_replaced("old-file", 100, true));
assert!(!cursor.reset_if_replaced("old-file", 150, true));
assert_eq!(cursor.byte_offset, 100, "an append keeps its offset");
}
#[test]
fn cursor_truncation_rewinds_from_previous_offset() {
let mut cursor = CursorState {
byte_offset: 100,
file_identity: Some("old-file".into()),
};
assert!(cursor.reset_if_replaced("old-file", 50, true));
assert_eq!(cursor.byte_offset, 0, "truncation restarts from zero");
}
#[test]
fn cursor_identity_change_rewinds_without_truncation() {
for backfill in [true, false] {
let mut cursor = CursorState {
byte_offset: 100,
file_identity: Some("old-file".into()),
};
assert!(cursor.reset_if_replaced("new-file", 150, backfill));
assert_eq!(
cursor.byte_offset, 0,
"a replacement at least as long as the offset restarts from zero"
);
assert_eq!(cursor.file_identity.as_deref(), Some("new-file"));
}
}
#[cfg(unix)]
#[tokio::test]
async fn same_size_atomic_replacement_is_reingested_after_cursor_reload() {
let (rt, dir) = runtime_without_schema();
apply_session_schema(&rt).await;
let path = dir.path().join("transcript.jsonl");
let original = concat!(
r#"{"uuid":"event-a","sessionId":"sess-replace","type":"user","timestamp":"2026-08-05T10:00:00Z","message":{"role":"user","content":"first"}}"#,
"\n"
);
let replacement = original.replace("event-a", "event-b");
assert_eq!(original.len(), replacement.len());
std::fs::write(&path, original).expect("original transcript");
let first = mirror_file(&rt, &path, 0, LineTailSource::ClaudeCode, None)
.await
.expect("first ingest");
assert_eq!(first.inserted, 1);
let mut cursor = load_cursors(&rt)
.await
.expect("reload cursor")
.remove(&path)
.expect("stored cursor");
assert_eq!(cursor.byte_offset, original.len() as u64);
assert_eq!(
cursor.file_identity.as_deref(),
Some(
file_identity(&std::fs::File::open(&path).expect("open original"))
.expect("original identity")
.as_str()
)
);
assert!(!cursor.reset_if_replaced(
&file_identity(&std::fs::File::open(&path).expect("open unchanged"))
.expect("unchanged identity"),
original.len() as u64,
true,
));
let staged = dir.path().join("replacement.jsonl");
std::fs::write(&staged, replacement).expect("replacement transcript");
std::fs::rename(&staged, &path).expect("atomic replacement");
let new_identity = file_identity(&std::fs::File::open(&path).expect("open replacement"))
.expect("replacement identity");
assert!(cursor.reset_if_replaced(&new_identity, original.len() as u64, true));
assert_eq!(cursor.byte_offset, 0);
let second = mirror_file(
&rt,
&path,
cursor.byte_offset,
LineTailSource::ClaudeCode,
None,
)
.await
.expect("replacement ingest");
assert_eq!(second.inserted, 1);
let saved = load_cursors(&rt)
.await
.expect("reload replacement cursor")
.remove(&path)
.expect("replacement cursor");
assert_eq!(saved.byte_offset, original.len() as u64);
assert_eq!(saved.file_identity.as_deref(), Some(new_identity.as_str()));
}
#[tokio::test]
async fn legacy_cursor_schema_adds_identity_without_losing_offset() {
let (rt, _dir) = runtime_without_schema();
let mut writer = rt.sql().writer().await.expect("writer");
writer
.execute_script(
"CREATE TABLE session_mirror_cursor (file_path TEXT PRIMARY KEY, session_id TEXT, byte_offset INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL)"
.to_string(),
)
.await
.expect("legacy cursor table");
drop(writer);
let path = PathBuf::from("/projects/legacy.jsonl");
insert_cursor_row(&rt, &path, 123).await;
rt.backend()
.apply_pack_ddl_statements_with_columns(
&SESSION_SCHEMA_PLAN_STMTS,
&SESSION_SCHEMA_COLUMN_ADDITIONS,
)
.expect("pack column upgrade");
let mut cursor = load_cursors(&rt)
.await
.expect("load upgraded cursor")
.remove(&path)
.expect("legacy row retained");
assert_eq!(cursor.byte_offset, 123);
assert_eq!(cursor.file_identity, None);
assert!(cursor.reset_if_replaced("current-file", 123, true));
assert_eq!(cursor.byte_offset, 0, "legacy row must replay once");
}
#[tokio::test]
async fn legacy_cursor_without_backfill_keeps_skipped_prefix_and_persists_identity() {
let (rt, dir) = runtime_without_schema();
let mut writer = rt.sql().writer().await.expect("writer");
writer
.execute_script(
"CREATE TABLE session_mirror_cursor (file_path TEXT PRIMARY KEY, session_id TEXT, byte_offset INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL)"
.to_string(),
)
.await
.expect("legacy cursor table");
drop(writer);
let path = dir.path().join("legacy.jsonl");
let skipped = concat!(
r#"{"uuid":"event-before-opt-out","sessionId":"sess-legacy","type":"user","timestamp":"2026-08-05T10:00:00Z","message":{"role":"user","content":"skipped"}}"#,
"\n"
);
let appended = concat!(
r#"{"uuid":"event-after-opt-out","sessionId":"sess-legacy","type":"user","timestamp":"2026-08-05T10:01:00Z","message":{"role":"user","content":"mirrored"}}"#,
"\n"
);
std::fs::write(&path, format!("{skipped}{appended}")).expect("transcript");
insert_cursor_row(&rt, &path, skipped.len() as i64).await;
rt.backend()
.apply_pack_ddl_statements_with_columns(
&SESSION_SCHEMA_PLAN_STMTS,
&SESSION_SCHEMA_COLUMN_ADDITIONS,
)
.expect("pack column upgrade");
let mut cursor = load_cursors(&rt)
.await
.expect("load upgraded cursor")
.remove(&path)
.expect("legacy row retained");
let file_len = std::fs::metadata(&path).expect("transcript metadata").len();
let identity = file_identity(&std::fs::File::open(&path).expect("open transcript"))
.expect("transcript identity");
assert!(
!reconcile_cursor_identity(&rt, &path, &mut cursor, &identity, file_len, false)
.await
.expect("adopt legacy identity")
);
assert_eq!(cursor.byte_offset, skipped.len() as u64);
assert_eq!(cursor.file_identity.as_deref(), Some(identity.as_str()));
let saved = load_cursors(&rt)
.await
.expect("reload adopted cursor")
.remove(&path)
.expect("adopted row");
assert_eq!(saved, cursor);
let mirrored = mirror_file(
&rt,
&path,
cursor.byte_offset,
LineTailSource::ClaudeCode,
None,
)
.await
.expect("tail after skipped prefix");
assert_eq!(mirrored.inserted, 1, "only the appended event is imported");
let mut truncated_legacy = CursorState {
byte_offset: skipped.len() as u64,
file_identity: None,
};
assert!(truncated_legacy.reset_if_replaced(&identity, 1, false));
assert_eq!(truncated_legacy.byte_offset, 0);
}
#[tokio::test]
async fn failed_cursor_delete_stays_pending_and_is_retried_next_tick() {
let (rt, _dir) = runtime_without_schema();
let discovery = DiscoveryIndex::default();
let path = PathBuf::from("/projects/gone.jsonl");
let mut pending = VecDeque::new();
queue_cursor_deletes(&mut pending, std::slice::from_ref(&path));
assert_eq!(pending.len(), 1);
let mut offsets = HashMap::new();
drain_pending_cursor_deletes(&rt, &discovery, &mut offsets, &mut pending).await;
assert_eq!(
pending.len(),
1,
"failed delete must remain in the retry set"
);
apply_session_schema(&rt).await;
insert_cursor_row(&rt, &path, 4096).await;
drain_pending_cursor_deletes(&rt, &discovery, &mut offsets, &mut pending).await;
assert!(pending.is_empty(), "successful retry drains the set");
assert!(
!cursor_row_exists(&rt, &path).await,
"stale row deleted on retry"
);
}
#[tokio::test]
async fn retracked_path_cancels_its_pending_cursor_delete() {
let (rt, _dir) = runtime_without_schema();
apply_session_schema(&rt).await;
let path = PathBuf::from("/projects/recreated.jsonl");
insert_cursor_row(&rt, &path, 512).await;
let mut discovery = DiscoveryIndex::default();
let mut pending = VecDeque::new();
queue_cursor_deletes(&mut pending, std::slice::from_ref(&path));
let mut offsets = HashMap::new();
discovery.add_file(path.clone(), DiscoveredKind::ChatGptExport, false);
drain_pending_cursor_deletes(&rt, &discovery, &mut offsets, &mut pending).await;
assert!(pending.is_empty(), "re-tracked path cancels its delete");
assert!(cursor_row_exists(&rt, &path).await, "fresh row preserved");
assert_eq!(
offsets.get(&path).map(|cursor| cursor.byte_offset),
Some(512),
"canceling the delete restores the in-memory offset from the preserved row"
);
}
#[tokio::test]
async fn same_pass_remove_and_readd_restores_offset_from_preserved_cursor_row() {
let (rt, _dir) = runtime_without_schema();
apply_session_schema(&rt).await;
let path = PathBuf::from("/projects/flap.jsonl");
insert_cursor_row(&rt, &path, 4096).await;
let mut discovery = DiscoveryIndex::default();
discovery.add_file(path.clone(), DiscoveredKind::ChatGptExport, false);
discovery.remove_file(&path, false);
let removed = discovery.take_removed_files();
assert_eq!(removed, vec![path.clone()]);
let mut offsets = HashMap::from([(
path.clone(),
CursorState {
byte_offset: 4096,
file_identity: None,
},
)]);
for removed_path in &removed {
offsets.remove(removed_path);
}
let mut pending = VecDeque::new();
queue_cursor_deletes(&mut pending, &removed);
assert!(offsets.is_empty(), "removal dropped the in-memory offset");
discovery.add_file(path.clone(), DiscoveredKind::ChatGptExport, false);
drain_pending_cursor_deletes(&rt, &discovery, &mut offsets, &mut pending).await;
assert!(
pending.is_empty(),
"re-added path cancels its pending delete"
);
assert!(
cursor_row_exists(&rt, &path).await,
"the preserved cursor row survives the cancel"
);
assert_eq!(
offsets.get(&path).map(|cursor| cursor.byte_offset),
Some(4096),
"the cancel restores the in-memory offset from the preserved row, \
so backfill=false seeding never falls back to EOF and skips bytes"
);
}
#[tokio::test]
async fn failed_cancelled_cursor_restore_blocks_eof_seed_until_retry() {
let (rt, _dir) = runtime_without_schema();
let mut file = NamedTempFile::new().expect("tmpfile");
let prefix = b"already mirrored\n";
let new_line = br#"{"uuid":"uuid-restored","sessionId":"sess-restored","type":"user","timestamp":"2026-08-05T10:00:00Z","message":{"role":"user","content":"restored"}}"#;
file.write_all(prefix).expect("prefix");
file.write_all(new_line).expect("new line");
file.write_all(b"\n").expect("line terminator");
let path = file.path().to_path_buf();
let true_offset = prefix.len() as u64;
let file_len = std::fs::metadata(&path).expect("file metadata").len();
let mut discovery = DiscoveryIndex::default();
discovery.add_file(path.clone(), DiscoveredKind::ChatGptExport, false);
let gone = path.with_file_name("gone.jsonl");
let mut pending = VecDeque::new();
queue_cursor_deletes(&mut pending, &[path.clone(), gone.clone()]);
let mut offsets = HashMap::new();
let blocked =
drain_pending_cursor_deletes(&rt, &discovery, &mut offsets, &mut pending).await;
assert!(blocked.contains(path.as_path()));
assert!(pending.contains(&path));
assert!(
pending.contains(&gone),
"an unrelated failed delete remains pending"
);
let scheduled = discovery
.schedule_files()
.into_iter()
.filter(|file| !blocked.contains(file.path.as_path()))
.collect::<Vec<_>>();
assert!(scheduled.is_empty(), "failed restore must block this tick");
assert!(
!offsets.contains_key(&path),
"failed restore must not insert an EOF seed before retry"
);
apply_session_schema(&rt).await;
insert_cursor_row(&rt, &path, true_offset as i64).await;
let blocked =
drain_pending_cursor_deletes(&rt, &discovery, &mut offsets, &mut pending).await;
assert!(blocked.is_empty());
assert!(pending.is_empty());
assert_eq!(
offsets.get(&path).map(|cursor| cursor.byte_offset),
Some(true_offset)
);
let restored_offset = offsets
.entry(path.clone())
.or_insert(CursorState {
byte_offset: file_len,
file_identity: None,
})
.byte_offset;
assert_eq!(restored_offset, true_offset);
let stats = mirror_file(
&rt,
&path,
restored_offset,
LineTailSource::ClaudeCode,
None,
)
.await
.expect("mirror bytes after restored cursor");
assert_eq!(
stats.inserted, 1,
"bytes after the preserved cursor are mirrored"
);
}
#[tokio::test]
async fn failed_empty_advance_commit_counts_as_error_for_cold_demotion() {
let (rt, _dir) = runtime_without_schema();
let path = PathBuf::from("/projects/blank.jsonl");
let mut discovery = DiscoveryIndex::default();
discovery.add_file(path.clone(), DiscoveredKind::ChatGptExport, false);
let (stats, had_errors) = finalize_dispatch_stats(
&rt,
&path,
0,
false,
Some(MirrorStats {
inserted: 0,
scanned: 0,
new_offset: 64,
..Default::default()
}),
false,
)
.await;
assert!(stats.is_none(), "failed commit must hold back the offset");
assert!(had_errors, "failed commit must become an error poll");
for _ in 1..FILE_ERROR_POLLS_BEFORE_COLD {
assert!(!tally_dispatch_errors(
&mut discovery,
&path,
false,
had_errors
));
}
assert!(tally_dispatch_errors(
&mut discovery,
&path,
false,
had_errors
));
assert!(discovery.files[&path].cold);
}
#[tokio::test]
async fn oversized_skip_advance_commits_despite_a_sibling_candidate_error() {
let (rt, _dir) = runtime_without_schema();
apply_session_schema(&rt).await;
let path = PathBuf::from("/projects/oversized.jsonl");
let start_offset = 0u64;
let mut dispatch = CandidateDispatch::default();
assert!(!dispatch.record(
Ok(MirrorStats {
inserted: 0,
scanned: 0,
new_offset: 4096,
skipped_oversized_bytes: true,
replay_mismatches: 0,
file_identity: None,
}),
start_offset,
));
assert!(!dispatch.record(Err(RuntimeError::Internal("boom".into())), start_offset,));
let CandidateDispatch { stats, errors } = dispatch;
let had_errors = !errors.is_empty();
assert!(had_errors, "the sibling error must be recorded");
let (stats, had_errors) =
finalize_dispatch_stats(&rt, &path, start_offset, false, stats, had_errors).await;
let stats = stats.expect(
"a skipped-oversized-line advance must commit even when a sibling \
candidate errored on the same span",
);
assert_eq!(stats.new_offset, 4096);
assert!(
had_errors,
"the sibling's error is still reported for cold-demotion accounting"
);
assert_eq!(
cursor_row_offset(&rt, &path).await,
Some(4096),
"the cursor for the oversized file must have durably advanced, so the \
next pass starts past the discarded prefix instead of re-reading it"
);
}
#[tokio::test]
async fn mixed_oversized_skip_and_scanned_line_keeps_the_error_veto() {
let (rt, _dir) = runtime_without_schema();
apply_session_schema(&rt).await;
let path = PathBuf::from("/projects/mixed.jsonl");
let start_offset = 0u64;
let mut dispatch = CandidateDispatch::default();
assert!(!dispatch.record(
Ok(MirrorStats {
inserted: 0,
scanned: 1,
new_offset: 4200,
skipped_oversized_bytes: true,
replay_mismatches: 0,
file_identity: None,
}),
start_offset,
));
assert!(!dispatch.record(Err(RuntimeError::Internal("boom".into())), start_offset,));
let CandidateDispatch { stats, errors } = dispatch;
let (stats, had_errors) =
finalize_dispatch_stats(&rt, &path, start_offset, false, stats, !errors.is_empty())
.await;
assert!(
stats.is_none(),
"an advance that also scanned an ordinary line stays vetoed while a sibling errored"
);
assert!(had_errors);
assert_eq!(
cursor_row_offset(&rt, &path).await,
None,
"no cursor row may be written for a contested span"
);
}
#[tokio::test]
async fn queue_cursor_deletes_deduplicates_and_is_bounded() {
let mut pending = VecDeque::new();
let path = PathBuf::from("/projects/gone.jsonl");
queue_cursor_deletes(&mut pending, &[path.clone(), path.clone()]);
assert_eq!(pending.len(), 1, "duplicate removals collapse to one entry");
for index in 0..(CURSOR_DELETE_RETRY_LIMIT + 8) {
let extra = PathBuf::from(format!("/projects/extra-{index}.jsonl"));
queue_cursor_deletes(&mut pending, &[extra]);
}
assert_eq!(
pending.len(),
CURSOR_DELETE_RETRY_LIMIT,
"the retry set is bounded; oldest entries are evicted"
);
assert!(
!pending.contains(&path),
"the oldest entry is the one evicted under the bound"
);
}
#[tokio::test]
async fn delete_cursors_reports_per_path_failures_without_aborting_the_batch() {
let (rt, _dir) = runtime_without_schema();
let path = PathBuf::from("/projects/gone.jsonl");
let failed = delete_cursors(&rt, std::slice::from_ref(&path))
.await
.expect("writer acquisition succeeds even when the table is missing");
assert_eq!(
failed,
vec![path],
"the failing path is reported for retry instead of aborting the batch"
);
}
#[tokio::test]
async fn head_of_line_cursor_delete_failure_does_not_block_the_batch() {
let (rt, _dir) = runtime_without_schema();
let first = PathBuf::from("/projects/first.jsonl");
let second = PathBuf::from("/projects/second.jsonl");
let third = PathBuf::from("/projects/third.jsonl");
let failed = delete_cursors(&rt, &[first.clone(), second.clone(), third.clone()])
.await
.expect("writer acquisition succeeds even when the table is missing");
assert_eq!(
failed,
vec![first, second, third],
"every path is attempted past the first failure; the failed paths \
are retained for retry"
);
}
#[tokio::test]
async fn delete_cursors_drains_the_batch_when_every_delete_succeeds() {
let (rt, _dir) = runtime_without_schema();
apply_session_schema(&rt).await;
let first = PathBuf::from("/projects/a.jsonl");
let second = PathBuf::from("/projects/b.jsonl");
insert_cursor_row(&rt, &first, 10).await;
insert_cursor_row(&rt, &second, 20).await;
let failed = delete_cursors(&rt, &[first.clone(), second.clone()])
.await
.expect("delete batch");
assert!(failed.is_empty(), "no per-path failures on the happy path");
assert!(!cursor_row_exists(&rt, &first).await);
assert!(!cursor_row_exists(&rt, &second).await);
}
}
#[cfg(test)]
mod codex_filename_tests {
use super::extract_codex_session_id;
use std::path::Path;
#[test]
fn real_codex_filename_yields_uuid() {
let path =
Path::new("rollout-2025-11-11T08-32-36-019a731e-4a58-71b1-a71f-a8d2f9782113.jsonl");
assert_eq!(
extract_codex_session_id(path).as_deref(),
Some("019a731e-4a58-71b1-a71f-a8d2f9782113")
);
}
#[test]
fn timestamp_only_stem_is_rejected() {
let path = Path::new("rollout-2025-11-11T08-32-36.jsonl");
assert_eq!(extract_codex_session_id(path), None);
}
#[test]
fn invalid_hex_suffix_is_rejected() {
let path =
Path::new("rollout-2025-11-11T08-32-36-zzzzzzzz-zzzz-zzzz-zzzz-zzzzzzzzzzzz.jsonl");
assert_eq!(extract_codex_session_id(path), None);
}
#[test]
fn too_short_suffix_is_rejected() {
let path = Path::new("rollout-2025-11-11T08-32-36-aaaa-bbbb-cccc-dddd.jsonl");
assert_eq!(extract_codex_session_id(path), None);
}
#[test]
fn non_rollout_filename_is_rejected() {
let path = Path::new("not-a-rollout-file.jsonl");
assert_eq!(extract_codex_session_id(path), None);
}
}