use anyhow::Result;
use notify_debouncer_mini::DebouncedEventKind;
use std::collections::HashMap;
use std::sync::{Arc, RwLock as StdRwLock};
use tokio::sync::{mpsc, oneshot};
use super::db::SessionDb;
use super::home_dir;
use super::ops::{OpKind, OpsRegistry};
use super::parser::{self, DiscoveredFile, ParseResult};
use super::state::WebState;
#[derive(Debug, Clone, serde::Serialize)]
pub struct SyncStats {
pub synced: usize,
pub failed: usize,
pub skipped: usize,
pub total: usize,
pub aborted: bool,
}
enum SyncRequest {
Incremental {
reply: oneshot::Sender<Result<SyncStats>>,
ops: Option<OpsRegistry>,
},
Full {
reply: oneshot::Sender<Result<SyncStats>>,
ops: Option<OpsRegistry>,
},
Targeted {
paths: Vec<String>,
reply: oneshot::Sender<Result<SyncStats>>,
ops: Option<OpsRegistry>,
},
}
#[derive(Clone)]
pub struct SyncHandle {
tx: mpsc::Sender<SyncRequest>,
last_stats: Arc<StdRwLock<Option<SyncStats>>>,
}
impl SyncHandle {
pub async fn sync_incremental(&self) -> Result<SyncStats> {
self.sync_incremental_tracked(None).await
}
pub async fn sync_incremental_tracked(&self, ops: Option<OpsRegistry>) -> Result<SyncStats> {
let (reply_tx, reply_rx) = oneshot::channel();
self.tx
.send(SyncRequest::Incremental { reply: reply_tx, ops })
.await
.map_err(|_| anyhow::anyhow!("sync actor is shut down"))?;
reply_rx
.await
.map_err(|_| anyhow::anyhow!("sync actor dropped reply"))?
}
pub async fn sync_full(&self) -> Result<SyncStats> {
self.sync_full_tracked(None).await
}
pub async fn sync_full_tracked(&self, ops: Option<OpsRegistry>) -> Result<SyncStats> {
let (reply_tx, reply_rx) = oneshot::channel();
self.tx
.send(SyncRequest::Full { reply: reply_tx, ops })
.await
.map_err(|_| anyhow::anyhow!("sync actor is shut down"))?;
reply_rx
.await
.map_err(|_| anyhow::anyhow!("sync actor dropped reply"))?
}
pub async fn sync_targeted(&self, paths: Vec<String>, ops: Option<OpsRegistry>) -> Result<SyncStats> {
let (reply_tx, reply_rx) = oneshot::channel();
self.tx
.send(SyncRequest::Targeted { paths, reply: reply_tx, ops })
.await
.map_err(|_| anyhow::anyhow!("sync actor is shut down"))?;
reply_rx
.await
.map_err(|_| anyhow::anyhow!("sync actor dropped reply"))?
}
pub fn last_stats(&self) -> Option<SyncStats> {
self.last_stats.read().unwrap().clone()
}
}
pub fn spawn_sync_actor(db: SessionDb) -> SyncHandle {
let (tx, rx) = mpsc::channel(8);
let last_stats = Arc::new(StdRwLock::new(None));
tokio::spawn(run_sync_actor(db, rx, Arc::clone(&last_stats)));
SyncHandle { tx, last_stats }
}
async fn run_sync_actor(
db: SessionDb,
mut rx: mpsc::Receiver<SyncRequest>,
last_stats: Arc<StdRwLock<Option<SyncStats>>>,
) {
let mut engine = SyncEngine::new(db);
if let Err(e) = engine.load_skip_cache().await {
eprintln!("[wsb] sync actor: failed to load skip cache: {}", e);
}
while let Some(req) = rx.recv().await {
match req {
SyncRequest::Incremental { reply, ops } => {
let result = engine.sync_incremental(ops).await;
if let Ok(ref stats) = result {
*last_stats.write().unwrap() = Some(stats.clone());
}
let _ = reply.send(result);
}
SyncRequest::Full { reply, ops } => {
let result = engine.sync_full(ops).await;
if let Ok(ref stats) = result {
*last_stats.write().unwrap() = Some(stats.clone());
}
let _ = reply.send(result);
}
SyncRequest::Targeted { paths, reply, ops } => {
let result = engine.sync_targeted(paths, ops).await;
if let Ok(ref stats) = result {
let mut guard = last_stats.write().unwrap();
if let Some(ref mut prev) = *guard {
prev.synced += stats.synced;
prev.failed += stats.failed;
prev.skipped += stats.skipped;
} else {
*guard = Some(stats.clone());
}
}
let _ = reply.send(result);
}
}
}
}
struct SyncEngine {
db: SessionDb,
skip_cache: HashMap<String, i64>, }
impl SyncEngine {
fn new(db: SessionDb) -> Self {
Self {
db,
skip_cache: HashMap::new(),
}
}
async fn load_skip_cache(&mut self) -> Result<()> {
self.skip_cache = self.db.load_skip_cache().await?;
Ok(())
}
async fn sync_incremental(&mut self, ops: Option<OpsRegistry>) -> Result<SyncStats> {
let op_handle = if let Some(ref registry) = ops {
Some(registry.start_op(OpKind::Sync, "Sync Claude history").await)
} else {
None
};
let files = match tokio::task::spawn_blocking(parser::discover_claude_sessions).await {
Ok(Ok(f)) => f,
Ok(Err(e)) => {
if let Some(h) = op_handle {
h.fail(format!("discovery failed: {}", e)).await;
}
return Err(e);
}
Err(e) => {
let err = anyhow::anyhow!("discovery panicked: {}", e);
if let Some(h) = op_handle {
h.fail(format!("discovery panicked: {}", e)).await;
}
return Err(err);
}
};
let total = files.len();
let mut synced = 0;
let mut failed = 0;
let mut skipped = 0;
for (file_idx, file) in files.iter().enumerate() {
if let Some(ref h) = op_handle {
if total > 0 {
let progress = file_idx as f32 / total as f32;
h.progress_labeled(
progress,
format!("Claude history ({}/{})", file_idx + 1, total),
)
.await;
}
}
let path_str = file.path.to_string_lossy().to_string();
let stat_path = file.path.clone();
let metadata = match tokio::task::spawn_blocking(move || std::fs::metadata(&stat_path))
.await
{
Ok(Ok(m)) => m,
Ok(Err(e)) => {
eprintln!("[wsb] sync: stat error {}: {}", file.path.display(), e);
failed += 1;
continue;
}
Err(e) => {
eprintln!("[wsb] sync: stat panicked {}: {}", file.path.display(), e);
failed += 1;
continue;
}
};
let file_size = metadata.len() as i64;
let file_mtime = metadata
.modified()
.ok()
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
.map(|d| d.as_nanos() as i64)
.unwrap_or(0);
if let Some(&cached_mtime) = self.skip_cache.get(&path_str) {
if cached_mtime == file_mtime {
skipped += 1;
continue;
}
}
if let Ok(Some((stored_size, stored_mtime))) =
self.db.get_session_file_info(&file.session_id).await
{
if stored_size == file_size && stored_mtime == file_mtime {
skipped += 1;
continue;
}
}
if let Ok(true) = self.db.is_excluded(&file.session_id).await {
self.skip_cache.insert(path_str.clone(), file_mtime);
skipped += 1;
continue;
}
let incremental_result = self.try_incremental(&file, file_size, file_mtime).await;
if let Some(Ok(())) = incremental_result {
synced += 1;
self.skip_cache.remove(&path_str);
continue;
}
let parse_path = file.path.clone();
let parse_project = file.project.clone();
let parse_result = tokio::task::spawn_blocking(move || {
parser::parse_claude_session(&parse_path, &parse_project, "local")
})
.await;
match parse_result {
Ok(Ok(results)) => {
if results.is_empty() {
self.skip_cache.insert(path_str, file_mtime);
continue;
}
for result in &results {
if let Err(e) = self.write_result(result).await {
eprintln!("[wsb] sync: write error for {}: {}", result.session.id, e);
failed += 1;
continue;
}
}
synced += results.len();
self.skip_cache.remove(&path_str);
}
Ok(Err(e)) => {
eprintln!("[wsb] sync: parse error {}: {}", file.path.display(), e);
self.skip_cache.insert(path_str, file_mtime);
failed += 1;
}
Err(e) => {
eprintln!("[wsb] sync: parse panicked {}: {}", file.path.display(), e);
self.skip_cache.insert(path_str, file_mtime);
failed += 1;
}
}
}
if let Err(e) = self.db.link_subagent_sessions().await {
eprintln!("[wsb] sync: link subagent sessions: {}", e);
}
if let Err(e) = self.db.persist_skip_cache(&self.skip_cache).await {
eprintln!("[wsb] sync: persist skip cache: {}", e);
}
let stats = SyncStats {
synced,
failed,
skipped,
total,
aborted: false,
};
if let Some(h) = op_handle {
if failed > 0 && synced == 0 {
h.fail(format!("{} file(s) failed to sync", failed)).await;
} else {
h.complete().await;
}
}
Ok(stats)
}
async fn sync_targeted(&mut self, paths: Vec<String>, ops: Option<OpsRegistry>) -> Result<SyncStats> {
let op_handle = if let Some(ref registry) = ops {
Some(registry.start_op(OpKind::Sync, format!("Sync {} Claude session(s)", paths.len())).await)
} else {
None
};
let path_set: std::collections::HashSet<String> = paths.into_iter().collect();
let path_set_clone = path_set.clone();
let all_files = match tokio::task::spawn_blocking(parser::discover_claude_sessions).await {
Ok(Ok(f)) => f,
Ok(Err(e)) => {
if let Some(h) = op_handle { h.fail(format!("discovery failed: {}", e)).await; }
return Err(e);
}
Err(e) => {
let err = anyhow::anyhow!("discovery panicked: {}", e);
if let Some(h) = op_handle { h.fail(format!("discovery panicked: {}", e)).await; }
return Err(err);
}
};
let files: Vec<DiscoveredFile> = all_files
.into_iter()
.filter(|f| path_set_clone.contains(&f.path.to_string_lossy().to_string()))
.collect();
let total = files.len();
let mut synced = 0;
let mut failed = 0;
let mut skipped = 0;
for (file_idx, file) in files.iter().enumerate() {
if let Some(ref h) = op_handle {
if total > 0 {
let progress = file_idx as f32 / total as f32;
h.progress_labeled(progress, format!("Claude history ({}/{})", file_idx + 1, total)).await;
}
}
let path_str = file.path.to_string_lossy().to_string();
let stat_path = file.path.clone();
let metadata = match tokio::task::spawn_blocking(move || std::fs::metadata(&stat_path)).await {
Ok(Ok(m)) => m,
Ok(Err(e)) => { eprintln!("[wsb] sync: stat error {}: {}", file.path.display(), e); failed += 1; continue; }
Err(e) => { eprintln!("[wsb] sync: stat panicked {}: {}", file.path.display(), e); failed += 1; continue; }
};
let file_size = metadata.len() as i64;
let file_mtime = metadata.modified().ok()
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
.map(|d| d.as_nanos() as i64)
.unwrap_or(0);
if let Some(&cached_mtime) = self.skip_cache.get(&path_str) {
if cached_mtime == file_mtime { skipped += 1; continue; }
}
if let Ok(Some((stored_size, stored_mtime))) = self.db.get_session_file_info(&file.session_id).await {
if stored_size == file_size && stored_mtime == file_mtime { skipped += 1; continue; }
}
if let Ok(true) = self.db.is_excluded(&file.session_id).await {
self.skip_cache.insert(path_str.clone(), file_mtime);
skipped += 1;
continue;
}
let incremental_result = self.try_incremental(file, file_size, file_mtime).await;
if let Some(Ok(())) = incremental_result {
synced += 1;
self.skip_cache.remove(&path_str);
continue;
}
let parse_path = file.path.clone();
let parse_project = file.project.clone();
let parse_result = tokio::task::spawn_blocking(move || {
parser::parse_claude_session(&parse_path, &parse_project, "local")
}).await;
match parse_result {
Ok(Ok(results)) => {
if results.is_empty() {
self.skip_cache.insert(path_str, file_mtime);
continue;
}
for result in &results {
if let Err(e) = self.write_result(result).await {
eprintln!("[wsb] sync: write error for {}: {}", result.session.id, e);
failed += 1;
continue;
}
}
synced += results.len();
self.skip_cache.remove(&path_str);
}
Ok(Err(e)) => { eprintln!("[wsb] sync: parse error {}: {}", path_str, e); failed += 1; }
Err(e) => { eprintln!("[wsb] sync: spawn_blocking panicked {}: {}", path_str, e); failed += 1; }
}
}
if let Err(e) = self.db.persist_skip_cache(&self.skip_cache).await {
eprintln!("[wsb] sync: failed to persist skip cache: {}", e);
}
let stats = SyncStats { synced, failed, skipped, total, aborted: false };
if let Some(h) = op_handle { h.complete().await; }
Ok(stats)
}
async fn sync_full(&mut self, ops: Option<OpsRegistry>) -> Result<SyncStats> {
let op_handle = if let Some(ref registry) = ops {
Some(registry.start_op(OpKind::Sync, "Resync Claude history (full)").await)
} else {
None
};
self.skip_cache.clear();
if let Err(e) = self.db.persist_skip_cache(&HashMap::new()).await {
eprintln!("[wsb] resync: clear skip cache: {}", e);
}
let result = self.sync_incremental(None).await;
match &result {
Ok(_) => {
if let Some(h) = op_handle {
h.complete().await;
}
}
Err(e) => {
if let Some(h) = op_handle {
h.fail(format!("{:#}", e)).await;
}
}
}
result
}
async fn try_incremental(
&self,
file: &DiscoveredFile,
current_size: i64,
current_mtime: i64,
) -> Option<Result<()>> {
let (stored_size, _stored_mtime) =
match self.db.get_session_file_info(&file.session_id).await {
Ok(Some(info)) => info,
_ => return None,
};
if current_size <= stored_size {
return None;
}
let max_ordinal = match self.db.max_ordinal(&file.session_id).await {
Ok(Some(ord)) => ord,
_ => return None,
};
let session_count = match self
.db
.count_sessions_for_path(&file.path.to_string_lossy())
.await
{
Ok(count) => count,
Err(_) => return None,
};
if session_count != 1 {
return None; }
let inc_path = file.path.clone();
let inc_session_id = file.session_id.clone();
let parse_result = tokio::task::spawn_blocking(move || {
parser::parse_claude_session_from(&inc_path, stored_size, max_ordinal + 1, &inc_session_id)
})
.await;
let parse_result = match parse_result {
Ok(r) => r,
Err(e) => {
eprintln!(
"[wsb] incremental parse panicked for {}: {}",
file.session_id, e
);
return None;
}
};
match parse_result {
Ok((new_msgs, ended_at, bytes_consumed)) => {
if new_msgs.is_empty() && bytes_consumed > 0 {
if let Err(e) = self
.db
.update_session_incremental(
&file.session_id,
ended_at.as_deref(),
-1,
-1,
stored_size + bytes_consumed,
current_mtime,
0,
0,
)
.await
{
return Some(Err(e));
}
return Some(Ok(()));
}
if new_msgs.is_empty() {
return None;
}
if let Err(e) = self.db.insert_messages(&new_msgs).await {
return Some(Err(e));
}
let mut total_output: i64 = 0;
let mut peak_context: i64 = 0;
let mut new_user_count: i64 = 0;
for msg in &new_msgs {
total_output += msg.output_tokens;
if msg.context_tokens > peak_context {
peak_context = msg.context_tokens;
}
if msg.role == "user" {
new_user_count += 1;
}
}
let existing_user_count = self
.db
.get_session_user_count(&file.session_id)
.await
.unwrap_or(0);
let new_offset = stored_size + bytes_consumed;
if let Err(e) = self
.db
.update_session_incremental(
&file.session_id,
ended_at.as_deref(),
max_ordinal + 1 + new_msgs.len() as i64,
existing_user_count + new_user_count,
new_offset,
current_mtime,
total_output,
peak_context,
)
.await
{
return Some(Err(e));
}
Some(Ok(()))
}
Err(_) => {
None
}
}
}
async fn write_result(&self, result: &ParseResult) -> Result<()> {
self.db.upsert_session(&result.session).await?;
if !result.messages.is_empty() {
self.db.insert_messages(&result.messages).await?;
}
Ok(())
}
}
pub async fn start_watcher(state: WebState) -> Result<()> {
use notify_debouncer_mini::{new_debouncer, notify::RecursiveMode, DebouncedEvent};
use std::time::Duration;
let home = home_dir()?;
let watch_dir = home.join(".claude").join("projects");
if !watch_dir.exists() {
eprintln!(
"[wsb] watch dir {} does not exist, file watcher disabled",
watch_dir.display()
);
return Ok(());
}
let (tx, mut rx) = tokio::sync::mpsc::channel(256);
let runtime_handle = tokio::runtime::Handle::current();
let watch_dir_clone = watch_dir.clone();
std::thread::spawn(move || {
let rt = runtime_handle;
let mut debouncer = match new_debouncer(
Duration::from_millis(500),
move |events: Result<Vec<DebouncedEvent>, notify::Error>| match events {
Ok(events) => {
let paths = extract_changed_jsonl_paths(events);
if !paths.is_empty() {
let tx = tx.clone();
rt.spawn(async move {
let _ = tx.send(paths).await;
});
}
}
Err(err) => {
eprintln!("[wsb] file watcher event error: {}", err);
}
},
) {
Ok(debouncer) => debouncer,
Err(err) => {
eprintln!("[wsb] failed to create file watcher: {}", err);
return;
}
};
if let Err(err) = debouncer
.watcher()
.watch(&watch_dir_clone, RecursiveMode::Recursive)
{
eprintln!(
"[wsb] failed to watch directory {}: {}",
watch_dir_clone.display(),
err
);
return;
}
loop {
std::thread::sleep(Duration::from_secs(3600));
}
});
while let Some(changed_paths) = rx.recv().await {
if let Err(e) = state.sync.sync_targeted(changed_paths, Some(state.ops.clone())).await {
eprintln!("[wsb] watcher sync failed: {}", e);
}
}
Ok(())
}
fn extract_changed_jsonl_paths(
events: Vec<notify_debouncer_mini::DebouncedEvent>,
) -> Vec<String> {
let mut paths = Vec::new();
for event in events {
if event.kind != DebouncedEventKind::Any
&& event.kind != DebouncedEventKind::AnyContinuous
{
continue;
}
if event.path.extension().and_then(|ext| ext.to_str()) != Some("jsonl") {
continue;
}
paths.push(event.path.to_string_lossy().to_string());
}
paths.sort();
paths.dedup();
paths
}
#[cfg(test)]
mod tests {
use super::*;
use notify_debouncer_mini::{DebouncedEvent, DebouncedEventKind};
use std::path::PathBuf;
#[test]
fn extract_changed_jsonl_paths_keeps_only_jsonl_and_deduplicates() {
let paths = extract_changed_jsonl_paths(vec![
DebouncedEvent {
path: PathBuf::from("/tmp/a.jsonl"),
kind: DebouncedEventKind::Any,
},
DebouncedEvent {
path: PathBuf::from("/tmp/a.jsonl"),
kind: DebouncedEventKind::AnyContinuous,
},
DebouncedEvent {
path: PathBuf::from("/tmp/b.txt"),
kind: DebouncedEventKind::Any,
},
DebouncedEvent {
path: PathBuf::from("/tmp/c.jsonl"),
kind: DebouncedEventKind::AnyContinuous,
},
]);
assert_eq!(
paths,
vec!["/tmp/a.jsonl".to_string(), "/tmp/c.jsonl".to_string(),]
);
}
#[tokio::test]
async fn sync_handle_last_stats_reads_shared_value() {
let (tx, _rx) = tokio::sync::mpsc::channel::<SyncRequest>(8);
let last_stats = std::sync::Arc::new(std::sync::RwLock::new(Some(SyncStats {
synced: 42,
failed: 0,
skipped: 0,
total: 42,
aborted: false,
})));
let handle = SyncHandle {
tx,
last_stats: std::sync::Arc::clone(&last_stats),
};
let mut futures = Vec::new();
for _ in 0..8 {
let h = handle.clone();
futures.push(tokio::spawn(async move { h.last_stats() }));
}
for fut in futures {
let stats = fut.await.expect("task did not panic");
let stats = stats.expect("should return Some");
assert_eq!(stats.synced, 42);
}
}
#[tokio::test]
async fn last_stats_does_not_block_on_spawn_blocking() {
let (tx, _rx) = tokio::sync::mpsc::channel::<SyncRequest>(8);
let last_stats = std::sync::Arc::new(std::sync::RwLock::new(Some(SyncStats {
synced: 1,
failed: 0,
skipped: 0,
total: 1,
aborted: false,
})));
let handle = SyncHandle { tx, last_stats };
tokio::spawn(async move {
tokio::task::spawn_blocking(|| {
std::thread::sleep(std::time::Duration::from_millis(200));
})
.await
.expect("spawn_blocking must not panic");
});
tokio::task::yield_now().await;
let start = std::time::Instant::now();
let stats = handle.last_stats();
let elapsed = start.elapsed();
assert!(stats.is_some(), "must return the cached stats");
assert!(
elapsed < std::time::Duration::from_millis(10),
"last_stats took {:?} — RwLock read should be near-instant",
elapsed
);
}
}