1use rusqlite::Connection;
2use sha2::Digest;
3use std::path::{Path, PathBuf};
4use std::process::Command;
5use thiserror::Error;
6use tracing::{info, warn};
7
8use crate::queries;
9use crate::workspace::Workspace;
10
11#[derive(Debug, Error)]
12pub enum SyncError {
13 #[error(
14 "julie-extract not found. Put it on PATH or set JULIE_EXTRACT_BIN. Download: https://github.com/anortham/julie-extractors/releases"
15 )]
16 BinaryNotFound,
17 #[error("Extractor failed with status {0}: {1}")]
18 ExtractionFailed(i32, String),
19 #[error("IO error during synchronization: {0}")]
20 Io(#[from] std::io::Error),
21 #[error("Database error during synchronization: {0}")]
22 Db(#[from] rusqlite::Error),
23 #[error("Database error: {0}")]
24 DbInit(#[from] crate::db::DbError),
25 #[error("workspace traversal failed: {0}")]
26 Walk(#[from] ignore::Error),
27}
28
29pub const PINNED_JULIE_VERSION: &str = "3.0.0";
30
31pub const EXTRACTION_LEVEL: &str = "facts";
34
35static CACHED_JULIE_BIN: std::sync::OnceLock<Option<(PathBuf, String)>> =
36 std::sync::OnceLock::new();
37
38pub fn find_julie_extract_binary() -> Option<PathBuf> {
43 installed_extractor().map(|(bin, _)| bin)
44}
45
46pub fn installed_extractor_version() -> String {
48 installed_extractor()
49 .map(|(_, version)| version)
50 .unwrap_or_else(|| PINNED_JULIE_VERSION.to_string())
51}
52
53fn installed_extractor() -> Option<(PathBuf, String)> {
54 CACHED_JULIE_BIN
55 .get_or_init(|| {
56 let candidates: Vec<(PathBuf, String)> = julie_extract_candidates()
57 .into_iter()
58 .filter_map(|bin| extractor_version(&bin).map(|version| (bin, version)))
59 .collect();
60 let pinned = candidates
61 .iter()
62 .find(|(_, version)| version == PINNED_JULIE_VERSION)
63 .cloned();
64 if pinned.is_some() {
65 return pinned;
66 }
67 let first = candidates.into_iter().next()?;
68 tracing::warn!(
69 found = %first.1,
70 pinned = %PINNED_JULIE_VERSION,
71 binary = %first.0.display(),
72 "julie-extract version differs from pinned version; AST facts may drift"
73 );
74 Some(first)
75 })
76 .clone()
77}
78
79fn extractor_version(bin: &Path) -> Option<String> {
80 let output = Command::new(bin).arg("--version").output().ok()?;
81 let text = String::from_utf8_lossy(&output.stdout);
82 text.split_whitespace().last().map(str::to_string)
83}
84
85fn julie_extract_candidates() -> Vec<PathBuf> {
86 let exe_name = if cfg!(windows) {
87 "julie-extract.exe"
88 } else {
89 "julie-extract"
90 };
91 let mut candidates = Vec::new();
92
93 if let Ok(path_str) = std::env::var("JULIE_EXTRACT_BIN") {
94 candidates.push(PathBuf::from(path_str));
95 }
96
97 if let Some(parent) = std::env::current_exe()
98 .ok()
99 .and_then(|p| p.parent().map(|d| d.to_path_buf()))
100 {
101 candidates.push(parent.join(exe_name));
102 candidates.push(parent.join(".tools").join(exe_name));
103 }
104
105 if let Ok(cwd) = std::env::current_dir() {
106 let mut probe = cwd;
107 loop {
108 candidates.push(probe.join(".tools").join(exe_name));
109 match probe.parent() {
110 Some(parent) if parent != probe => probe = parent.to_path_buf(),
111 _ => break,
112 }
113 }
114 }
115
116 if let Ok(p) = which::which("julie-extract") {
117 candidates.push(p);
118 }
119
120 candidates
121 .into_iter()
122 .filter(|p| p.is_file())
123 .map(|p| crate::workspace::normalize_path(&p))
124 .collect()
125}
126
127pub fn execute_julie_extract(args: &[&str]) -> Result<String, SyncError> {
129 let bin = find_julie_extract_binary().ok_or(SyncError::BinaryNotFound)?;
130
131 let mut attempts = 0;
132 loop {
133 let output = Command::new(&bin)
134 .args(args)
135 .output()
136 .map_err(SyncError::Io)?;
137
138 if output.status.success() {
139 return Ok(String::from_utf8_lossy(&output.stdout).to_string());
140 }
141
142 let stderr = String::from_utf8_lossy(&output.stderr).to_string();
143 if (stderr.contains("database is locked")
145 || stderr.contains("busy")
146 || stderr.contains("SQLITE_BUSY"))
147 && attempts < 5
148 {
149 attempts += 1;
150 std::thread::sleep(std::time::Duration::from_millis(50 * (1 << attempts)));
151 continue;
152 }
153
154 let code = output.status.code().unwrap_or(-1);
155 return Err(SyncError::ExtractionFailed(code, stderr));
156 }
157}
158
159pub fn update_file(workspace: &Workspace, db_path: &Path, rel_path: &str) -> Result<(), SyncError> {
161 let root_str = workspace.canonical_root.to_string_lossy();
162 let db_str = db_path.to_string_lossy();
163
164 execute_julie_extract(&[
165 "update", "--root", &root_str, "--db", &db_str, "--file", rel_path,
166 ])?;
167
168 Ok(())
169}
170
171pub fn delete_file(workspace: &Workspace, db_path: &Path, rel_path: &str) -> Result<(), SyncError> {
173 let root_str = workspace.canonical_root.to_string_lossy();
174 let db_str = db_path.to_string_lossy();
175
176 execute_julie_extract(&[
177 "delete", "--root", &root_str, "--db", &db_str, "--file", rel_path,
178 ])?;
179
180 Ok(())
181}
182
183pub fn scan_workspace(workspace: &Workspace, db_path: &Path, force: bool) -> Result<(), SyncError> {
188 let root_str = workspace.canonical_root.to_string_lossy();
189 let db_str = db_path.to_string_lossy();
190 let own_pid = std::process::id().to_string();
191
192 if let Some(parent) = db_path.parent() {
193 std::fs::create_dir_all(parent)?;
194 }
195
196 let scan_args = |new_artifact: bool| {
197 let mut args = vec!["scan", "--root", &*root_str, "--db", &*db_str];
198 if new_artifact {
199 args.extend(["--level", EXTRACTION_LEVEL]);
200 }
201 if cfg!(unix) {
202 args.extend(["--parent-pid", &*own_pid]);
203 }
204 if force {
205 args.push("--force");
206 }
207 args
208 };
209
210 match execute_julie_extract(&scan_args(!db_path.exists())) {
211 Ok(_) => {}
212 Err(SyncError::ExtractionFailed(_, stderr))
213 if stderr.contains("schema_incompatible") && db_path.exists() =>
214 {
215 warn!("Extractor cannot read the existing artifact; rebuilding from scratch");
216 remove_artifact_files(db_path)?;
217 execute_julie_extract(&scan_args(true))?;
218 }
219 Err(e) => return Err(e),
220 }
221
222 let _ = crate::db::ensure_fts_index_path(db_path);
224
225 Ok(())
226}
227
228pub fn ensure_index_matches_extractor(
232 workspace: &Workspace,
233 db_path: &Path,
234 extractor_version: &str,
235) -> Result<bool, SyncError> {
236 if !db_path.exists() {
237 return Ok(false);
238 }
239 let metadata = |key: &str| -> Option<String> {
240 let conn = crate::db::open_read_only(db_path).ok()?;
241 conn.query_row(
242 "SELECT value FROM artifact_metadata WHERE key = ?1",
243 [key],
244 |r| r.get(0),
245 )
246 .ok()
247 };
248 let Some(recorded) = metadata("binary_version") else {
249 return Ok(false);
250 };
251 let level = metadata("index_level").unwrap_or_else(|| "full".to_string());
252 if recorded == extractor_version && level == EXTRACTION_LEVEL {
253 return Ok(false);
254 }
255 info!(
256 recorded = %recorded,
257 installed = %extractor_version,
258 level = %level,
259 wanted_level = %EXTRACTION_LEVEL,
260 "Index was written by a different julie-extract version or level; rebuilding"
261 );
262 remove_artifact_files(db_path)?;
263 scan_workspace(workspace, db_path, true)?;
264 Ok(true)
265}
266
267fn remove_artifact_files(db_path: &Path) -> Result<(), SyncError> {
268 for suffix in ["", "-wal", "-shm"] {
269 let sidecar = PathBuf::from(format!("{}{suffix}", db_path.display()));
270 match std::fs::remove_file(&sidecar) {
271 Ok(()) => {}
272 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
273 Err(e) => return Err(e.into()),
274 }
275 }
276 Ok(())
277}
278
279pub fn compute_content_hash_matches(disk_bytes: &[u8], stored_hash: &str) -> bool {
281 if stored_hash.starts_with("blake3:") {
282 let b3 = format!("blake3:{}", blake3::hash(disk_bytes).to_hex());
283 b3 == stored_hash
284 } else {
285 let b3 = blake3::hash(disk_bytes).to_hex().to_string();
286 if b3 == stored_hash {
287 return true;
288 }
289 let mut hasher = sha2::Sha256::new();
290 sha2::Digest::update(&mut hasher, disk_bytes);
291 let sha = hex::encode(sha2::Digest::finalize(hasher));
292 sha == stored_hash || format!("sha256:{sha}") == stored_hash
293 }
294}
295
296pub fn ensure_fresh_file(
300 workspace: &Workspace,
301 db_path: &Path,
302 conn: &Connection,
303 rel_path: &str,
304) -> Result<bool, SyncError> {
305 let abs_path = workspace.canonical_root.join(rel_path);
306 let meta = match std::fs::metadata(&abs_path) {
307 Ok(meta) => meta,
308 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
309 let existing_file = queries::get_file(conn, rel_path).map_err(|e| match e {
310 queries::QueryError::Sqlite(err) => SyncError::Db(err),
311 _ => SyncError::Db(rusqlite::Error::QueryReturnedNoRows),
312 })?;
313 if existing_file.is_some() {
314 delete_file(workspace, db_path, rel_path)?;
315 return Ok(true);
316 }
317 return Ok(false);
318 }
319 Err(error) => return Err(SyncError::Io(error)),
320 };
321
322 if meta.is_dir() {
323 return Ok(false);
324 }
325
326 let disk_bytes = meta.len() as i64;
327
328 let existing_file = queries::get_file(conn, rel_path).map_err(|e| match e {
330 queries::QueryError::Sqlite(err) => SyncError::Db(err),
331 _ => SyncError::Db(rusqlite::Error::QueryReturnedNoRows),
332 })?;
333
334 let is_dirty = match existing_file {
335 None => true, Some(f) => {
337 if f.content_bytes != disk_bytes {
339 true
340 } else {
341 let disk_content = std::fs::read(&abs_path)?;
342 !compute_content_hash_matches(&disk_content, &f.content_hash)
343 }
344 }
345 };
346
347 if is_dirty {
348 let stored_root: Option<String> = conn
349 .query_row(
350 "SELECT value FROM artifact_metadata WHERE key = 'root_path'",
351 [],
352 |r| r.get(0),
353 )
354 .ok();
355 if let Some(r) = stored_root
356 && !crate::workspace::paths_equal(Path::new(&r), &workspace.canonical_root)
357 {
358 crate::db::retarget_artifact_root(db_path, &workspace.canonical_root)?;
359 }
360
361 update_file(workspace, db_path, rel_path)?;
362 return Ok(true);
363 }
364
365 Ok(false)
366}
367
368#[derive(Debug, Default)]
370pub struct ReconcileReport {
371 pub added: Vec<String>,
372 pub modified: Vec<String>,
373 pub deleted: Vec<String>,
374}
375
376fn extract_error_path(err: &ignore::Error) -> Option<&Path> {
377 match err {
378 ignore::Error::WithPath { path, .. } => Some(path.as_path()),
379 ignore::Error::WithDepth { err, .. } => extract_error_path(err),
380 ignore::Error::WithLineNumber { err, .. } => extract_error_path(err),
381 ignore::Error::Loop { ancestor, .. } => Some(ancestor.as_path()),
382 _ => None,
383 }
384}
385
386pub fn create_index(workspace: &Workspace, db_path: &Path) -> Result<(), SyncError> {
392 if let Some(parent_db) = parent_repository_db(&workspace.canonical_root)
393 && copy_parent_index(workspace, db_path, &parent_db)
394 {
395 return Ok(());
396 }
397 scan_workspace(workspace, db_path, false)
398}
399
400fn parent_repository_db(root: &Path) -> Option<PathBuf> {
401 let git_marker = root.join(".git");
402 if !git_marker.is_file() {
403 return None;
404 }
405 let git_content = std::fs::read_to_string(&git_marker).ok()?;
406 let gitdir = git_content
407 .lines()
408 .find_map(|l| l.strip_prefix("gitdir:"))?
409 .trim();
410 let gitdir = Path::new(gitdir);
411 let mut probe = if gitdir.is_absolute() {
412 gitdir.to_path_buf()
413 } else {
414 root.join(gitdir)
415 };
416 while let Some(parent) = probe.parent() {
417 if parent == probe {
418 break;
419 }
420 if parent.join(".git").exists() {
421 let db = parent.join(".code-kb").join("artifact.db");
422 return db.exists().then_some(db);
423 }
424 probe = parent.to_path_buf();
425 }
426 None
427}
428
429fn copy_parent_index(workspace: &Workspace, db_path: &Path, parent_db: &Path) -> bool {
430 let flushed = crate::db::open_read_write(parent_db)
431 .map(|conn| crate::db::checkpoint_truncate(&conn).is_ok())
432 .unwrap_or(false);
433 if !flushed {
434 return false;
435 }
436 if let Some(dir) = db_path.parent() {
437 let _ = std::fs::create_dir_all(dir);
438 }
439 if std::fs::copy(parent_db, db_path).is_err() {
440 return false;
441 }
442 info!(from = %parent_db.display(), to = %db_path.display(), "Worktree fast-path: copied parent database, reconciling");
443 let reconciled = crate::db::retarget_artifact_root(db_path, &workspace.canonical_root)
444 .and_then(|_| crate::db::ensure_fts_index_path(db_path))
445 .is_ok()
446 && crate::db::open_read_only(db_path)
447 .ok()
448 .and_then(|conn| reconcile_offline_edits(workspace, db_path, &conn).ok())
449 .is_some();
450 if !reconciled {
451 warn!(
452 "Failed to retarget worktree database root; removing copied db and falling back to full scan"
453 );
454 let _ = std::fs::remove_file(db_path);
455 }
456 reconciled
457}
458
459pub fn reconcile_offline_edits(
460 workspace: &Workspace,
461 db_path: &Path,
462 conn: &Connection,
463) -> Result<ReconcileReport, SyncError> {
464 let stored_root: Option<String> = conn
467 .query_row(
468 "SELECT value FROM artifact_metadata WHERE key = 'root_path'",
469 [],
470 |r| r.get(0),
471 )
472 .ok();
473 if let Some(r) = stored_root
474 && !crate::workspace::paths_equal(Path::new(&r), &workspace.canonical_root)
475 {
476 crate::db::retarget_artifact_root(db_path, &workspace.canonical_root)?;
477 }
478
479 let mut report = ReconcileReport::default();
480
481 let temp_conn = Connection::open_in_memory().map_err(SyncError::Db)?;
483 temp_conn
484 .execute(
485 "CREATE TABLE _seen (path TEXT COLLATE NOCASE PRIMARY KEY)",
486 [],
487 )
488 .map_err(SyncError::Db)?;
489
490 let mut insert_seen_stmt = temp_conn
491 .prepare("INSERT OR IGNORE INTO _seen (path) VALUES (?1)")
492 .map_err(SyncError::Db)?;
493
494 let mut check_file_stmt = conn
495 .prepare("SELECT content_bytes, content_hash FROM files WHERE path = ?1")
496 .map_err(SyncError::Db)?;
497
498 let mut walker = ignore::WalkBuilder::new(&workspace.canonical_root);
499 walker
500 .standard_filters(true)
501 .hidden(false)
502 .add_custom_ignore_filename(".julieignore")
503 .add_custom_ignore_filename(".code-kb-ignore")
504 .add_custom_ignore_filename(".codekbignore")
505 .filter_entry(|entry| {
506 let name = entry.file_name().to_string_lossy();
507 !crate::workspace::is_hard_excluded(&name)
508 });
509 let walker = walker.build();
510
511 temp_conn
512 .execute("BEGIN TRANSACTION", [])
513 .map_err(SyncError::Db)?;
514
515 let mut unreadable_prefixes: Vec<String> = Vec::new();
516
517 for result in walker {
518 let entry = match result {
519 Ok(e) => e,
520 Err(e) => {
521 warn!("Reconciliation walker encountered error: {e}");
522 if let Some(path) = extract_error_path(&e) {
523 let norm_path = dunce::simplified(path);
524 if let Some(rel) =
525 crate::workspace::strip_prefix_lossy(norm_path, &workspace.canonical_root)
526 {
527 unreadable_prefixes.push(crate::workspace::to_forward_slash(rel));
528 }
529 }
530 continue;
531 }
532 };
533
534 if entry.file_type().map(|t| t.is_file()).unwrap_or(false) {
535 let path = entry.path();
536 let norm_path = dunce::simplified(path);
537 if let Some(rel) =
538 crate::workspace::strip_prefix_lossy(norm_path, &workspace.canonical_root)
539 {
540 let rel_str = crate::workspace::to_forward_slash(rel);
541 if crate::workspace::is_hard_excluded(&rel_str) {
542 continue;
543 }
544 let bytes = match entry.metadata() {
545 Ok(m) => m.len() as i64,
546 Err(e) => {
547 warn!("Failed to read metadata for '{}': {e}", path.display());
548 let _ = insert_seen_stmt.execute([&rel_str]);
549 continue;
550 }
551 };
552
553 insert_seen_stmt
554 .execute([&rel_str])
555 .map_err(SyncError::Db)?;
556
557 let mut rows = check_file_stmt.query([&rel_str]).map_err(SyncError::Db)?;
558
559 if let Some(row) = rows.next().map_err(SyncError::Db)? {
560 let indexed_bytes: i64 = row.get(0).map_err(SyncError::Db)?;
561 let stored_hash: String = row.get(1).map_err(SyncError::Db)?;
562
563 let is_modified = if indexed_bytes != bytes {
564 true
565 } else {
566 match std::fs::read(path) {
567 Ok(content) => !compute_content_hash_matches(&content, &stored_hash),
568 Err(e) => {
569 warn!(
570 "Failed to read '{}' for hash verification: {e}",
571 path.display()
572 );
573 false
574 }
575 }
576 };
577
578 if is_modified {
579 report.modified.push(rel_str);
580 }
581 } else {
582 report.added.push(rel_str);
583 }
584 }
585 }
586 }
587
588 temp_conn.execute("COMMIT", []).map_err(SyncError::Db)?;
589
590 let mut files_stmt = conn
591 .prepare("SELECT path FROM files")
592 .map_err(SyncError::Db)?;
593
594 let mut exists_seen_stmt = temp_conn
595 .prepare("SELECT 1 FROM _seen WHERE path = ?1")
596 .map_err(SyncError::Db)?;
597
598 let mut file_rows = files_stmt.query([]).map_err(SyncError::Db)?;
599 while let Some(row) = file_rows.next().map_err(SyncError::Db)? {
600 let indexed_path: String = row.get(0).map_err(SyncError::Db)?;
601
602 let in_unreadable_prefix = unreadable_prefixes.iter().any(|prefix| {
604 indexed_path == *prefix || indexed_path.starts_with(&format!("{prefix}/"))
605 });
606 if in_unreadable_prefix {
607 continue;
608 }
609
610 let mut seen_rows = exists_seen_stmt
611 .query([&indexed_path])
612 .map_err(SyncError::Db)?;
613 if seen_rows.next().map_err(SyncError::Db)?.is_none() {
614 report.deleted.push(indexed_path);
615 }
616 }
617
618 drop(file_rows);
619 drop(files_stmt);
620 drop(check_file_stmt);
621 drop(exists_seen_stmt);
622 drop(insert_seen_stmt);
623 drop(temp_conn);
624
625 let total_changes = report.added.len() + report.modified.len() + report.deleted.len();
626 if total_changes > 0 {
627 info!(
628 "Cold start reconciliation detected {} changes (+{}, ~{}, -{})",
629 total_changes,
630 report.added.len(),
631 report.modified.len(),
632 report.deleted.len()
633 );
634
635 if total_changes > 50 {
636 scan_workspace(workspace, db_path, false)?;
638 } else {
639 for added in &report.added {
641 if let Err(e) = update_file(workspace, db_path, added) {
642 warn!("Failed to index added file '{}': {e}", added);
643 }
644 }
645 for modified in &report.modified {
646 if let Err(e) = update_file(workspace, db_path, modified) {
647 warn!("Failed to index modified file '{}': {e}", modified);
648 }
649 }
650 for deleted in &report.deleted {
651 if let Err(e) = delete_file(workspace, db_path, deleted) {
652 warn!("Failed to remove deleted file '{}': {e}", deleted);
653 }
654 }
655 }
656 }
657
658 Ok(report)
659}
660
661#[cfg(test)]
662mod tests {
663 use super::*;
664
665 #[test]
666 fn test_find_julie_extract_binary() {
667 let path = find_julie_extract_binary()
668 .expect("julie-extract binary must be present for tests (see scripts/julie-pins.json)");
669 assert!(path.exists(), "Discovered path must exist: {:?}", path);
670 }
671
672 #[test]
673 fn test_reconcile_offline_edits_drive_case_mismatch() {
674 let temp = crate::safe_tempdir();
675 let db_path = temp.path().join("test.db");
676 let conn = rusqlite::Connection::open(&db_path).unwrap();
677 conn.execute_batch(
678 "CREATE TABLE files (
679 file_id TEXT PRIMARY KEY,
680 path TEXT NOT NULL,
681 language TEXT,
682 content_hash TEXT,
683 content_bytes INTEGER,
684 line_count INTEGER,
685 indexed_at TEXT
686 );
687 CREATE TABLE symbols (
688 symbol_id TEXT PRIMARY KEY,
689 file_id TEXT,
690 path TEXT NOT NULL
691 );",
692 )
693 .unwrap();
694
695 let src_dir = temp.path().join("src");
697 std::fs::create_dir_all(&src_dir).unwrap();
698 let file_path = src_dir.join("main.rs");
699 let content = "fn main() {}\n";
700 std::fs::write(&file_path, content).unwrap();
701
702 let hash = sha2::Sha256::digest(content.as_bytes());
703 let hash_hex = hex::encode(hash);
704
705 conn.execute(
706 "INSERT INTO files VALUES ('f1', 'src/main.rs', 'rust', ?1, ?2, 1, '2026-09-14T00:00:00Z')",
707 rusqlite::params![hash_hex, content.len() as i64],
708 )
709 .unwrap();
710
711 #[allow(unused_mut)]
712 let mut ws = Workspace::new(temp.path().to_path_buf());
713 #[cfg(windows)]
714 {
715 let root_str = ws.canonical_root.to_string_lossy().to_string();
716 if let Some(first_char) = root_str.chars().next() {
717 let flipped = if first_char.is_ascii_uppercase() {
718 first_char.to_ascii_lowercase()
719 } else {
720 first_char.to_ascii_uppercase()
721 };
722 let altered_root = format!("{}{}", flipped, &root_str[1..]);
723 ws.canonical_root = PathBuf::from(altered_root);
724 }
725 }
726
727 let report = reconcile_offline_edits(&ws, &db_path, &conn).unwrap();
728 assert!(
729 report.deleted.is_empty(),
730 "Files should not be marked deleted due to drive casing difference: {:?}",
731 report.deleted
732 );
733 }
734}