1use std::io;
31use std::path::Path;
32use std::time::Instant;
33
34use anyhow::{Context, Result};
35use rusqlite::Connection;
36
37use crate::db::{migrations, project, query, rebuild};
38use crate::event::Event;
39use crate::shard::ShardManager;
40
41#[derive(Debug, Clone, PartialEq, Eq)]
48pub struct EventHash(pub String);
49
50#[derive(Debug, Clone)]
52pub struct ApplyReport {
53 pub events_applied: usize,
55 pub shards_scanned: usize,
57 pub full_rebuild_triggered: bool,
59 pub full_rebuild_reason: Option<String>,
61 pub elapsed: std::time::Duration,
63}
64
65#[allow(clippy::too_many_lines)]
91pub fn incremental_apply(
92 events_dir: &Path,
93 db_path: &Path,
94 force_full: bool,
95) -> Result<ApplyReport> {
96 let start = Instant::now();
97
98 if force_full {
99 return do_full_rebuild(events_dir, db_path, start, "force_full flag set");
100 }
101
102 let Some(conn) = query::try_open_projection_raw(db_path)? else {
106 return do_full_rebuild(
107 events_dir,
108 db_path,
109 start,
110 "projection database missing or corrupt",
111 );
112 };
113
114 let (byte_offset, last_hash) =
116 query::get_projection_cursor(&conn).context("read projection cursor")?;
117
118 if byte_offset == 0 && last_hash.is_none() {
120 drop(conn);
121 return do_full_rebuild(events_dir, db_path, start, "fresh database (no cursor)");
122 }
123
124 if let Err(reason) = check_incremental_safety(&conn, events_dir) {
126 drop(conn);
127 return do_full_rebuild(events_dir, db_path, start, &reason);
128 }
129
130 let bones_dir = events_dir.parent().unwrap_or_else(|| Path::new("."));
132 let shard_mgr = ShardManager::new(bones_dir);
133 let shards = shard_mgr
134 .list_shards()
135 .map_err(|e| anyhow::anyhow!("list shards: {e}"))?;
136 let shards_scanned = shards.len();
137
138 let offset = usize::try_from(byte_offset).unwrap_or(0);
139
140 if let Some(ref hash) = last_hash {
143 let tail_ok = validate_cursor_hash_at_offset(&shard_mgr, offset, hash).unwrap_or(false);
144 if !tail_ok {
145 drop(conn);
146 return do_full_rebuild(
147 events_dir,
148 db_path,
149 start,
150 "cursor hash not found at expected byte offset",
151 );
152 }
153 }
154
155 let mut line_iter = shard_mgr
156 .replay_lines_from_offset(offset)
157 .map_err(|e| anyhow::anyhow!("open shard line iterator: {e}"))?
158 .peekable();
159
160 if line_iter.peek().is_none() {
162 return Ok(ApplyReport {
163 events_applied: 0,
164 shards_scanned,
165 full_rebuild_triggered: false,
166 full_rebuild_reason: None,
167 elapsed: start.elapsed(),
168 });
169 }
170
171 project::ensure_tracking_table(&conn).context("ensure projected_events tracking table")?;
173
174 let mut version_checked = false;
175 let mut shard_version = crate::event::parser::CURRENT_VERSION;
176 let mut line_no = 0;
177 let mut total_projected = 0;
178 let mut total_duplicates = 0;
179 let mut total_errors = 0;
180 let mut current_last_hash = last_hash;
181 let mut total_byte_len = offset;
182
183 let mut current_batch: Vec<Event> = Vec::with_capacity(1000);
184 let projector = project::Projector::new(&conn);
185
186 for line_res in line_iter {
187 let (abs_offset, line): (usize, String) =
188 line_res.map_err(|e: io::Error| anyhow::anyhow!("read shard line: {e}"))?;
189 line_no += 1;
190 total_byte_len = abs_offset + line.len();
191
192 if !version_checked && line.trim_start().starts_with("# bones event log v") {
194 version_checked = true;
195 shard_version = crate::event::parser::detect_version(&line)
196 .map_err(|msg| anyhow::anyhow!("version check failed: {msg}"))?;
197 continue;
198 }
199
200 match crate::event::parser::parse_line(&line) {
201 Ok(crate::event::parser::ParsedLine::Event(event)) => {
202 let event = crate::event::migrate_event(*event, shard_version)
203 .map_err(|e| anyhow::anyhow!("migration failed: {e}"))?;
204
205 current_last_hash = Some(event.event_hash.clone());
206 current_batch.push(event);
207
208 if current_batch.len() >= 1000 {
209 let stats = projector
210 .project_batch(¤t_batch)
211 .context("project batch during incremental apply")?;
212 total_projected += stats.projected;
213 total_duplicates += stats.duplicates;
214 total_errors += stats.errors;
215 current_batch.clear();
216 }
217 }
218 Ok(
219 crate::event::parser::ParsedLine::Comment(_)
220 | crate::event::parser::ParsedLine::Blank,
221 ) => {}
222 Err(crate::event::parser::ParseError::InvalidEventType(raw)) => {
223 tracing::warn!(line = line_no, event_type = %raw, "skipping unknown event type");
224 }
225 Err(e) => anyhow::bail!("parse error at line {line_no} (offset {abs_offset}): {e}"),
226 }
227 }
228
229 if !current_batch.is_empty() {
231 let stats = projector
232 .project_batch(¤t_batch)
233 .context("project final batch during incremental apply")?;
234 total_projected += stats.projected;
235 total_duplicates += stats.duplicates;
236 total_errors += stats.errors;
237 }
238
239 let new_offset = i64::try_from(total_byte_len).unwrap_or(i64::MAX);
241 query::update_projection_cursor(&conn, new_offset, current_last_hash.as_deref())
242 .context("update projection cursor after incremental apply")?;
243
244 tracing::info!(
245 events_applied = total_projected,
246 duplicates = total_duplicates,
247 errors = total_errors,
248 shards_scanned,
249 byte_offset_from = byte_offset,
250 byte_offset_to = new_offset,
251 elapsed_ms = start.elapsed().as_millis(),
252 "incremental projection apply complete"
253 );
254
255 Ok(ApplyReport {
256 events_applied: total_projected,
257 shards_scanned,
258 full_rebuild_triggered: false,
259 full_rebuild_reason: None,
260 elapsed: start.elapsed(),
261 })
262}
263
264pub fn read_hwm(db: &Connection) -> Result<Option<EventHash>> {
271 let (_offset, hash) = query::get_projection_cursor(db).context("read high-water mark")?;
272 Ok(hash.map(EventHash))
273}
274
275pub fn write_hwm(db: &Connection, hwm: &EventHash) -> Result<()> {
281 let (offset, _) =
283 query::get_projection_cursor(db).context("read current cursor for hwm update")?;
284 query::update_projection_cursor(db, offset, Some(&hwm.0)).context("write high-water mark")?;
285 Ok(())
286}
287
288pub fn check_incremental_safety(db: &Connection, events_dir: &Path) -> Result<(), String> {
303 let schema_version = migrations::current_schema_version(db)
305 .map_err(|e| format!("failed to read schema version: {e}"))?;
306 if schema_version != migrations::LATEST_SCHEMA_VERSION {
307 return Err(format!(
308 "schema version mismatch: db has v{schema_version}, code expects v{}",
309 migrations::LATEST_SCHEMA_VERSION
310 ));
311 }
312
313 let table_exists: bool = db
315 .query_row(
316 "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='projected_events')",
317 [],
318 |row| row.get(0),
319 )
320 .map_err(|e| format!("failed to check projected_events table: {e}"))?;
321 if !table_exists {
322 return Err("projected_events tracking table missing".into());
323 }
324
325 let bones_dir = events_dir.parent().unwrap_or_else(|| Path::new("."));
327 let shard_mgr = ShardManager::new(bones_dir);
328 let shards = shard_mgr
329 .list_shards()
330 .map_err(|e| format!("failed to list shards: {e}"))?;
331
332 if shards.len() > 1 {
334 for &(year, month) in &shards[..shards.len() - 1] {
335 if let Ok(Some(manifest)) = shard_mgr.read_manifest(year, month) {
336 let shard_path = shard_mgr.shard_path(year, month);
337 match std::fs::metadata(&shard_path) {
338 Ok(meta) => {
339 if meta.len() != manifest.byte_len {
340 return Err(format!(
341 "sealed shard {}-{:02} size mismatch: \
342 manifest says {} bytes, file is {} bytes",
343 year,
344 month,
345 manifest.byte_len,
346 meta.len()
347 ));
348 }
349 }
350 Err(e) => {
351 return Err(format!("cannot stat sealed shard {year}-{month:02}: {e}"));
352 }
353 }
354 }
355 }
358 }
359
360 Ok(())
361}
362
363fn validate_cursor_hash_at_offset(
373 shard_mgr: &ShardManager,
374 offset: usize,
375 hash: &str,
376) -> Result<bool> {
377 if offset == 0 {
378 return Ok(false);
379 }
380 let search_start = offset.saturating_sub(512);
381 let window = shard_mgr
382 .read_content_range(search_start, offset)
383 .map_err(|e| anyhow::anyhow!("read cursor hash window: {e}"))?;
384 Ok(window.contains(hash))
385}
386
387#[cfg(test)]
391fn validate_cursor_hash(content: &str, offset: usize, hash: &str) -> bool {
392 if offset == 0 || offset > content.len() {
393 return false;
394 }
395
396 let before = &content[..offset];
397 let search_start = offset.saturating_sub(512);
398 let search_region = &before[search_start..];
399 search_region.contains(hash)
400}
401
402fn do_full_rebuild(
404 events_dir: &Path,
405 db_path: &Path,
406 start: Instant,
407 reason: &str,
408) -> Result<ApplyReport> {
409 tracing::info!(reason, "falling back to full projection rebuild");
410
411 let report = rebuild::rebuild(events_dir, db_path)
412 .context("full rebuild during incremental apply fallback")?;
413
414 Ok(ApplyReport {
415 events_applied: report.event_count,
416 shards_scanned: report.shard_count,
417 full_rebuild_triggered: true,
418 full_rebuild_reason: Some(reason.to_string()),
419 elapsed: start.elapsed(),
420 })
421}
422
423#[cfg(test)]
428mod tests {
429 use super::*;
430 use crate::db::open_projection;
431 use crate::event::Event;
432 use crate::event::data::*;
433 use crate::event::types::EventType;
434 use crate::event::writer;
435 use crate::model::item::{Kind, Size, Urgency};
436 use crate::model::item_id::ItemId;
437 use std::collections::BTreeMap;
438 use tempfile::TempDir;
439
440 fn setup_bones_dir() -> (TempDir, ShardManager) {
445 let dir = TempDir::new().expect("create tempdir");
446 let shard_mgr = ShardManager::new(dir.path());
447 shard_mgr.ensure_dirs().expect("ensure dirs");
448 shard_mgr.init().expect("init shard");
449 (dir, shard_mgr)
450 }
451
452 fn make_create_event(id: &str, title: &str, ts: i64) -> Event {
453 let mut event = Event {
454 wall_ts_us: ts,
455 agent: "test-agent".into(),
456 itc: "itc:AQ".into(),
457 parents: vec![],
458 event_type: EventType::Create,
459 item_id: ItemId::new_unchecked(id),
460 data: EventData::Create(CreateData {
461 title: title.into(),
462 kind: Kind::Task,
463 size: Some(Size::M),
464 urgency: Urgency::Default,
465 labels: vec!["test".into()],
466 parent: None,
467 causation: None,
468 description: Some(format!("Description for {title}")),
469 extra: BTreeMap::new(),
470 }),
471 event_hash: String::new(),
472 };
473 writer::write_event(&mut event).expect("compute hash");
474 event
475 }
476
477 fn append_event(shard_mgr: &ShardManager, event: &Event) {
478 let line = writer::write_line(event).expect("serialize event");
479 let (year, month) = shard_mgr.active_shard().unwrap().unwrap();
480 shard_mgr
481 .append_raw(year, month, &line)
482 .expect("append event");
483 }
484
485 #[test]
490 fn incremental_apply_on_empty_db_does_full_rebuild() {
491 let (dir, _shard_mgr) = setup_bones_dir();
492 let db_path = dir.path().join("bones.db");
493 let events_dir = dir.path().join("events");
494
495 let report = incremental_apply(&events_dir, &db_path, false).unwrap();
496 assert!(report.full_rebuild_triggered);
497 assert!(
498 report
499 .full_rebuild_reason
500 .as_deref()
501 .unwrap()
502 .contains("missing"),
503 "reason: {:?}",
504 report.full_rebuild_reason
505 );
506 }
507
508 #[test]
509 fn incremental_apply_force_full() {
510 let (dir, shard_mgr) = setup_bones_dir();
511 let db_path = dir.path().join("bones.db");
512 let events_dir = dir.path().join("events");
513
514 let create = make_create_event("bn-001", "Item 1", 1000);
515 append_event(&shard_mgr, &create);
516
517 rebuild::rebuild(&events_dir, &db_path).unwrap();
519
520 let report = incremental_apply(&events_dir, &db_path, true).unwrap();
522 assert!(report.full_rebuild_triggered);
523 assert_eq!(
524 report.full_rebuild_reason.as_deref(),
525 Some("force_full flag set")
526 );
527 assert_eq!(report.events_applied, 1);
528 }
529
530 #[test]
531 fn incremental_apply_picks_up_new_events() {
532 let (dir, shard_mgr) = setup_bones_dir();
533 let db_path = dir.path().join("bones.db");
534 let events_dir = dir.path().join("events");
535
536 let create1 = make_create_event("bn-001", "Item 1", 1000);
538 let create2 = make_create_event("bn-002", "Item 2", 1001);
539 append_event(&shard_mgr, &create1);
540 append_event(&shard_mgr, &create2);
541
542 rebuild::rebuild(&events_dir, &db_path).unwrap();
543
544 let create3 = make_create_event("bn-003", "Item 3", 1002);
546 append_event(&shard_mgr, &create3);
547
548 let report = incremental_apply(&events_dir, &db_path, false).unwrap();
550 assert!(!report.full_rebuild_triggered);
551 assert_eq!(report.events_applied, 1);
552
553 let conn = open_projection(&db_path).unwrap();
555 let count: i64 = conn
556 .query_row("SELECT COUNT(*) FROM items", [], |row| row.get(0))
557 .unwrap();
558 assert_eq!(count, 3);
559 }
560
561 #[test]
562 fn incremental_apply_noop_when_up_to_date() {
563 let (dir, shard_mgr) = setup_bones_dir();
564 let db_path = dir.path().join("bones.db");
565 let events_dir = dir.path().join("events");
566
567 let create = make_create_event("bn-001", "Item 1", 1000);
568 append_event(&shard_mgr, &create);
569
570 rebuild::rebuild(&events_dir, &db_path).unwrap();
571
572 let report = incremental_apply(&events_dir, &db_path, false).unwrap();
574 assert!(!report.full_rebuild_triggered);
575 assert_eq!(report.events_applied, 0);
576 }
577
578 #[test]
579 fn incremental_apply_multiple_rounds() {
580 let (dir, shard_mgr) = setup_bones_dir();
581 let db_path = dir.path().join("bones.db");
582 let events_dir = dir.path().join("events");
583
584 let e1 = make_create_event("bn-001", "Item 1", 1000);
586 append_event(&shard_mgr, &e1);
587 rebuild::rebuild(&events_dir, &db_path).unwrap();
588
589 let e2 = make_create_event("bn-002", "Item 2", 1001);
591 append_event(&shard_mgr, &e2);
592 let r2 = incremental_apply(&events_dir, &db_path, false).unwrap();
593 assert!(!r2.full_rebuild_triggered);
594 assert_eq!(r2.events_applied, 1);
595
596 let e3 = make_create_event("bn-003", "Item 3", 1002);
598 let e4 = make_create_event("bn-004", "Item 4", 1003);
599 append_event(&shard_mgr, &e3);
600 append_event(&shard_mgr, &e4);
601 let r3 = incremental_apply(&events_dir, &db_path, false).unwrap();
602 assert!(!r3.full_rebuild_triggered);
603 assert_eq!(r3.events_applied, 2);
604
605 let conn = open_projection(&db_path).unwrap();
607 let count: i64 = conn
608 .query_row("SELECT COUNT(*) FROM items", [], |row| row.get(0))
609 .unwrap();
610 assert_eq!(count, 4);
611 }
612
613 #[test]
614 fn incremental_apply_matches_full_rebuild() {
615 let (dir, shard_mgr) = setup_bones_dir();
616 let events_dir = dir.path().join("events");
617
618 for i in 0..10 {
620 let e = make_create_event(
621 &format!("bn-{i:03x}"),
622 &format!("Item {i}"),
623 1000 + i64::from(i),
624 );
625 append_event(&shard_mgr, &e);
626 }
627
628 let db_full = dir.path().join("full.db");
630 rebuild::rebuild(&events_dir, &db_full).unwrap();
631
632 let db_inc = dir.path().join("inc.db");
634 rebuild::rebuild(&events_dir, &db_inc).unwrap();
638
639 let conn_full = open_projection(&db_full).unwrap();
641 let conn_inc = open_projection(&db_inc).unwrap();
642
643 let count_full: i64 = conn_full
644 .query_row("SELECT COUNT(*) FROM items", [], |row| row.get(0))
645 .unwrap();
646 let count_inc: i64 = conn_inc
647 .query_row("SELECT COUNT(*) FROM items", [], |row| row.get(0))
648 .unwrap();
649 assert_eq!(count_full, count_inc);
650 assert_eq!(count_full, 10);
651
652 let titles_full: Vec<String> = {
654 let mut stmt = conn_full
655 .prepare("SELECT title FROM items ORDER BY item_id")
656 .unwrap();
657 stmt.query_map([], |row| row.get::<_, String>(0))
658 .unwrap()
659 .map(|r| r.unwrap())
660 .collect()
661 };
662 let titles_inc: Vec<String> = {
663 let mut stmt = conn_inc
664 .prepare("SELECT title FROM items ORDER BY item_id")
665 .unwrap();
666 stmt.query_map([], |row| row.get::<_, String>(0))
667 .unwrap()
668 .map(|r| r.unwrap())
669 .collect()
670 };
671 assert_eq!(titles_full, titles_inc);
672 }
673
674 #[test]
675 fn schema_version_mismatch_triggers_full_rebuild() {
676 let (dir, shard_mgr) = setup_bones_dir();
677 let db_path = dir.path().join("bones.db");
678 let events_dir = dir.path().join("events");
679
680 let create = make_create_event("bn-001", "Item 1", 1000);
681 append_event(&shard_mgr, &create);
682
683 rebuild::rebuild(&events_dir, &db_path).unwrap();
684
685 {
687 let conn = open_projection(&db_path).unwrap();
688 conn.pragma_update(None, "user_version", 999_i64).unwrap();
689 }
690
691 let report = incremental_apply(&events_dir, &db_path, false).unwrap();
692 assert!(report.full_rebuild_triggered);
693 assert!(
694 report
695 .full_rebuild_reason
696 .as_deref()
697 .unwrap()
698 .contains("schema version"),
699 "reason: {:?}",
700 report.full_rebuild_reason
701 );
702 }
703
704 #[test]
705 fn read_hwm_returns_none_for_fresh_db() {
706 let mut conn = Connection::open_in_memory().unwrap();
707 migrations::migrate(&mut conn).unwrap();
708
709 let hwm = read_hwm(&conn).unwrap();
710 assert!(hwm.is_none());
711 }
712
713 #[test]
714 fn write_and_read_hwm_roundtrip() {
715 let mut conn = Connection::open_in_memory().unwrap();
716 migrations::migrate(&mut conn).unwrap();
717
718 let hash = EventHash("blake3:abc123".into());
719 write_hwm(&conn, &hash).unwrap();
720
721 let retrieved = read_hwm(&conn).unwrap();
722 assert_eq!(retrieved.unwrap(), hash);
723 }
724
725 #[test]
726 fn check_incremental_safety_passes_valid_db() {
727 let (dir, shard_mgr) = setup_bones_dir();
728 let db_path = dir.path().join("bones.db");
729 let events_dir = dir.path().join("events");
730
731 let create = make_create_event("bn-001", "Item 1", 1000);
732 append_event(&shard_mgr, &create);
733
734 rebuild::rebuild(&events_dir, &db_path).unwrap();
735
736 let conn = open_projection(&db_path).unwrap();
737 project::ensure_tracking_table(&conn).unwrap();
738 let result = check_incremental_safety(&conn, &events_dir);
739 assert!(result.is_ok(), "safety check failed: {result:?}");
740 }
741
742 #[test]
743 fn check_incremental_safety_fails_schema_mismatch() {
744 let mut conn = Connection::open_in_memory().unwrap();
745 migrations::migrate(&mut conn).unwrap();
746 conn.pragma_update(None, "user_version", 999_i64).unwrap();
747
748 let result = check_incremental_safety(&conn, Path::new("/nonexistent"));
750 assert!(result.is_err());
751 assert!(result.unwrap_err().contains("schema version"));
752 }
753
754 #[test]
755 fn check_incremental_safety_fails_missing_tracking_table() {
756 let mut conn = Connection::open_in_memory().unwrap();
757 migrations::migrate(&mut conn).unwrap();
758 let result = check_incremental_safety(&conn, Path::new("/nonexistent"));
761 assert!(result.is_err());
762 assert!(result.unwrap_err().contains("projected_events"));
763 }
764
765 #[test]
766 fn validate_cursor_hash_finds_hash_near_offset() {
767 let content = "line1\thash1\nline2\tblake3:abc123\nline3\thash3\n";
768 let offset = content.find("line3").unwrap();
769 assert!(validate_cursor_hash(content, offset, "blake3:abc123"));
770 }
771
772 #[test]
773 fn validate_cursor_hash_fails_wrong_hash() {
774 let content = "line1\thash1\nline2\tblake3:abc123\nline3\thash3\n";
775 let offset = content.find("line3").unwrap();
776 assert!(!validate_cursor_hash(content, offset, "blake3:zzz999"));
777 }
778
779 #[test]
780 fn validate_cursor_hash_fails_zero_offset() {
781 let content = "line1\tblake3:abc123\n";
782 assert!(!validate_cursor_hash(content, 0, "blake3:abc123"));
783 }
784}