1use super::*;
4
5pub(super) fn open_queue_db<P: AsRef<std::path::Path>>(path: P) -> Result<Connection, AppError> {
20 let conn = Connection::open(path)?;
21 conn.pragma_update(None, "journal_mode", "wal")?;
22 conn.pragma_update(None, "busy_timeout", crate::constants::BUSY_TIMEOUT_MILLIS)?;
28 conn.execute_batch(
29 "CREATE TABLE IF NOT EXISTS queue (
30 id INTEGER PRIMARY KEY AUTOINCREMENT,
31 item_key TEXT NOT NULL UNIQUE,
32 item_type TEXT NOT NULL DEFAULT 'memory',
33 status TEXT NOT NULL DEFAULT 'pending',
34 memory_id INTEGER,
35 entity_id INTEGER,
36 entities INTEGER DEFAULT 0,
37 rels INTEGER DEFAULT 0,
38 error TEXT,
39 cost_usd REAL DEFAULT 0.0,
40 attempt INTEGER DEFAULT 0,
41 elapsed_ms INTEGER,
42 created_at TEXT DEFAULT (datetime('now')),
43 done_at TEXT
44 );
45 CREATE INDEX IF NOT EXISTS idx_enrich_queue_status ON queue(status);",
46 )?;
47 let mut has_error_class = false;
51 let mut has_next_retry_at = false;
52 let mut has_operation = false;
57 let mut has_finish_reason = false;
63 let mut has_input_tokens = false;
64 let mut has_output_tokens = false;
65 {
66 let mut stmt = conn.prepare("PRAGMA table_info(queue)")?;
67 let names = stmt.query_map([], |r| r.get::<_, String>(1))?;
68 for name in names {
69 match name?.as_str() {
70 "error_class" => has_error_class = true,
71 "next_retry_at" => has_next_retry_at = true,
72 "operation" => has_operation = true,
73 "finish_reason" => has_finish_reason = true,
74 "input_tokens" => has_input_tokens = true,
75 "output_tokens" => has_output_tokens = true,
76 _ => {}
77 }
78 }
79 }
80 if !has_error_class {
81 conn.execute_batch("ALTER TABLE queue ADD COLUMN error_class TEXT")?;
82 }
83 if !has_next_retry_at {
84 conn.execute_batch("ALTER TABLE queue ADD COLUMN next_retry_at TEXT")?;
85 }
86 if !has_operation {
87 conn.execute_batch("ALTER TABLE queue ADD COLUMN operation TEXT")?;
88 }
89 if !has_finish_reason {
90 conn.execute_batch("ALTER TABLE queue ADD COLUMN finish_reason TEXT")?;
91 }
92 if !has_input_tokens {
93 conn.execute_batch("ALTER TABLE queue ADD COLUMN input_tokens INTEGER")?;
94 }
95 if !has_output_tokens {
96 conn.execute_batch("ALTER TABLE queue ADD COLUMN output_tokens INTEGER")?;
97 }
98 conn.execute_batch(
99 "CREATE INDEX IF NOT EXISTS idx_enrich_queue_eligible ON queue(status, next_retry_at);
100 CREATE INDEX IF NOT EXISTS idx_enrich_queue_operation ON queue(operation, status);
101 CREATE INDEX IF NOT EXISTS idx_enrich_queue_memory ON queue(memory_id)",
102 )?;
103 Ok(conn)
104}
105
106pub(super) fn enqueue_candidate(
114 queue_conn: &Connection,
115 main_conn: &Connection,
116 namespace: &str,
117 key: &str,
118 item_type: &str,
119 operation: &str,
120) {
121 let memory_id: Option<i64> = if item_type == "memory" {
122 main_conn
123 .query_row(
124 "SELECT id FROM memories WHERE namespace=?1 AND name=?2 AND deleted_at IS NULL",
125 rusqlite::params![namespace, key],
126 |r| r.get(0),
127 )
128 .ok()
129 } else {
130 None
131 };
132 if let Err(e) = queue_conn.execute(
133 "INSERT OR IGNORE INTO queue (item_key, item_type, status, operation, memory_id) \
134 VALUES (?1, ?2, 'pending', ?3, ?4)",
135 rusqlite::params![key, item_type, operation, memory_id],
136 ) {
137 tracing::warn!(target: "enrich", error = %e, "queue insert failed");
138 }
139}
140
141pub(super) fn skipped_item_keys(
149 conn: &Connection,
150 operation: &str,
151) -> Result<std::collections::HashSet<String>, AppError> {
152 let mut stmt = conn.prepare(
153 "SELECT item_key FROM queue WHERE status='skipped' AND (operation = ?1 OR operation IS NULL)",
154 )?;
155 let keys = stmt
156 .query_map(rusqlite::params![operation], |r| r.get::<_, String>(0))?
157 .collect::<Result<std::collections::HashSet<String>, _>>()?;
158 Ok(keys)
159}
160
161pub(super) fn item_type_for(operation: &EnrichOperation) -> &'static str {
164 match operation {
165 EnrichOperation::EntityDescriptions => "entity",
166 _ => "memory",
167 }
168}
169
170pub(super) fn item_type_for_key(key: &str, default: &'static str) -> &'static str {
178 if key.starts_with("entity:") {
179 "entity"
180 } else if key.starts_with("chunk:") {
181 "chunk"
182 } else {
183 default
184 }
185}
186
187pub fn cleanup_queue_entry(db_path: &std::path::Path, memory_id: i64, name: &str) {
196 let queue_path = crate::paths::sidecar_path(db_path, ".enrich-queue.sqlite");
197 if !queue_path.exists() {
198 return;
199 }
200 match open_queue_db(&queue_path) {
201 Ok(conn) => {
202 if let Err(e) = conn.execute(
203 "DELETE FROM queue WHERE memory_id = ?1 OR item_key = ?2",
204 rusqlite::params![memory_id, name],
205 ) {
206 tracing::warn!(target: "enrich", error = %e, memory_id, "enrich-queue cleanup failed");
207 }
208 }
209 Err(e) => {
210 tracing::warn!(target: "enrich", error = %e, "enrich-queue cleanup skipped (open failed)");
211 }
212 }
213}
214
215pub(super) fn prune_dead_orphans(
226 queue_conn: &Connection,
227 main_conn: &Connection,
228 operation: &str,
229 namespace: &str,
230) -> Result<i64, AppError> {
231 let dead: Vec<(i64, String)> = {
232 let mut stmt = queue_conn.prepare(
233 "SELECT id, item_key FROM queue \
234 WHERE status='dead' AND item_type='memory' \
235 AND (operation = ?1 OR operation IS NULL) ORDER BY id",
236 )?;
237 let rows = stmt
238 .query_map(rusqlite::params![operation], |r| Ok((r.get(0)?, r.get(1)?)))?
239 .collect::<Result<Vec<_>, _>>()?;
240 rows
241 };
242 let mut pruned = 0_i64;
243 for (id, name) in dead {
244 let exists = main_conn
245 .query_row(
246 "SELECT 1 FROM memories WHERE namespace=?1 AND name=?2 AND deleted_at IS NULL",
247 rusqlite::params![namespace, name],
248 |_| Ok(()),
249 )
250 .is_ok();
251 if !exists {
252 queue_conn.execute("DELETE FROM queue WHERE id=?1", rusqlite::params![id])?;
253 pruned += 1;
254 }
255 }
256 if pruned > 0 {
257 let _ = queue_conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);");
258 }
259 Ok(pruned)
260}
261
262pub(super) fn prune_dead_entity_orphans(
268 queue_conn: &Connection,
269 operation: &str,
270) -> Result<i64, AppError> {
271 let pruned = queue_conn.execute(
272 "DELETE FROM queue \
273 WHERE status='dead' AND item_type='entity' \
274 AND (operation = ?1 OR operation IS NULL)",
275 rusqlite::params![operation],
276 )? as i64;
277 if pruned > 0 {
278 let _ = queue_conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);");
279 }
280 Ok(pruned)
281}
282
283#[derive(Debug, Serialize, schemars::JsonSchema)]
293pub struct EnrichStatus {
294 pub(super) status_report: bool,
295 pub(super) operation: String,
296 pub(super) namespace: String,
297 pub(super) unbound_backlog: usize,
298 pub(super) scan_backlog: i64,
305 pub(super) queue_pending: i64,
306 pub(super) queue_processing: i64,
307 pub(super) queue_done: i64,
308 pub(super) queue_failed: i64,
309 pub(super) queue_skipped: i64,
310 pub(super) queue_dead: i64,
311 pub(super) eligible_now: i64,
312 pub(super) waiting: i64,
313 pub(super) state: &'static str,
319 pub(super) waiting_items: Vec<WaitingItem>,
322}
323
324#[derive(Debug, Serialize, schemars::JsonSchema)]
326pub struct WaitingItem {
327 pub(super) item_key: String,
328 pub(super) attempt: i64,
329 pub(super) next_retry_at: Option<String>,
330 pub(super) error_class: Option<String>,
331}
332
333#[derive(Debug, Serialize, schemars::JsonSchema)]
335pub struct DeadItem {
336 pub(super) dead_item: bool,
337 pub(super) item_key: String,
338 pub(super) item_type: String,
339 pub(super) attempt: i64,
340 pub(super) error_class: Option<String>,
341 pub(super) error: Option<String>,
342 pub(super) finish_reason: Option<String>,
347 pub(super) input_tokens: Option<i64>,
349 pub(super) output_tokens: Option<i64>,
351}
352
353#[derive(Debug, Serialize, schemars::JsonSchema)]
355pub struct DeadSummary {
356 pub(super) summary: bool,
357 pub(super) operation: String,
358 pub(super) namespace: String,
359 pub(super) action: &'static str,
361 pub(super) dead_total: i64,
362 pub(super) requeued: i64,
363 pub(super) pruned: i64,
367}
368
369pub(super) fn classify_enrich_outcome(e: &AppError) -> crate::retry::AttemptOutcome {
380 use crate::retry::AttemptOutcome;
381 match e {
382 AppError::RateLimited { .. } | AppError::Timeout { .. } | AppError::DbBusy(_) => {
383 AttemptOutcome::Transient
384 }
385 AppError::EntityNotYetMaterialized { .. } => AttemptOutcome::Transient,
393 AppError::ProviderError { .. }
399 | AppError::NotFound(_)
400 | AppError::MemoryNotFound { .. }
401 | AppError::MemoryNotFoundById { .. } => AttemptOutcome::HardFailure,
402 AppError::Database(_) => {
406 if crate::storage::utils::is_sqlite_busy(e) {
407 AttemptOutcome::Transient
408 } else {
409 AttemptOutcome::HardFailure
410 }
411 }
412 AppError::Embedding(_) => AttemptOutcome::Transient,
423 _ => AttemptOutcome::HardFailure,
434 }
435}
436
437pub(super) fn record_item_failure(
450 queue_conn: &rusqlite::Connection,
451 queue_id: i64,
452 attempt: i64,
453 max_attempts: u32,
454 err: &AppError,
455) -> crate::retry::AttemptOutcome {
456 let outcome = classify_enrich_outcome(err);
457 let err_str = format!("{err}");
458 record_item_failure_typed(
459 queue_conn,
460 queue_id,
461 attempt,
462 max_attempts,
463 outcome,
464 &err_str,
465 None,
466 None,
467 None,
468 )
469}
470
471#[allow(clippy::too_many_arguments)]
480pub(super) fn record_item_failure_typed(
481 queue_conn: &rusqlite::Connection,
482 queue_id: i64,
483 attempt: i64,
484 max_attempts: u32,
485 outcome: crate::retry::AttemptOutcome,
486 err_str: &str,
487 finish_reason: Option<&str>,
488 input_tokens: Option<i64>,
489 output_tokens: Option<i64>,
490) -> crate::retry::AttemptOutcome {
491 use crate::retry::AttemptOutcome;
492 let error_class = match outcome {
493 AttemptOutcome::Transient => "transient",
494 AttemptOutcome::HardFailure => "permanent",
495 AttemptOutcome::Success => "success",
496 };
497
498 let terminal = matches!(outcome, AttemptOutcome::HardFailure) || attempt >= max_attempts as i64;
499 if terminal {
500 let _ = queue_conn.execute(
501 "UPDATE queue SET status='dead', error=?1, error_class=?2, done_at=datetime('now'), \
502 finish_reason=?3, input_tokens=?4, output_tokens=?5 WHERE id=?6",
503 rusqlite::params![
504 err_str,
505 error_class,
506 finish_reason,
507 input_tokens,
508 output_tokens,
509 queue_id
510 ],
511 );
512 } else {
513 let delay = crate::retry::compute_delay(
514 &crate::retry::RetryConfig::llm_rate_limit(),
515 attempt.max(0) as u32,
516 );
517 let secs = delay.as_secs().max(1);
518 let modifier = format!("+{secs} seconds");
519 let _ = queue_conn.execute(
520 "UPDATE queue SET status='pending', error=?1, error_class=?2, next_retry_at=datetime('now', ?3), \
521 finish_reason=?4, input_tokens=?5, output_tokens=?6 WHERE id=?7",
522 rusqlite::params![
523 err_str,
524 error_class,
525 modifier,
526 finish_reason,
527 input_tokens,
528 output_tokens,
529 queue_id
530 ],
531 );
532 }
533 outcome
534}
535
536pub(super) enum DequeueOutcome {
544 Claimed((i64, String, String, i64)),
545 Empty,
546}
547
548pub(super) fn dequeue_next_pending(
549 queue_conn: &rusqlite::Connection,
550 backoff_clause: &str,
551) -> Result<DequeueOutcome, AppError> {
552 let dequeue_sql = format!(
553 "UPDATE queue SET status='processing', attempt=attempt+1 \
554 WHERE id = (SELECT id FROM queue WHERE status='pending' {backoff_clause} \
555 ORDER BY id LIMIT 1) \
556 RETURNING id, item_key, item_type, attempt"
557 );
558 match queue_conn.query_row(&dequeue_sql, [], |row| {
559 Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?))
560 }) {
561 Ok(claimed) => Ok(DequeueOutcome::Claimed(claimed)),
562 Err(rusqlite::Error::QueryReturnedNoRows) => Ok(DequeueOutcome::Empty),
563 Err(e) => Err(AppError::Database(e)),
564 }
565}
566
567#[cfg(test)]
572mod tests {
573 use super::*;
574
575 fn open_test_db() -> Connection {
576 let conn = Connection::open_in_memory().expect("in-memory db");
577 conn.execute_batch(
578 "CREATE TABLE memories (
579 id INTEGER PRIMARY KEY AUTOINCREMENT,
580 namespace TEXT NOT NULL DEFAULT 'global',
581 name TEXT NOT NULL,
582 type TEXT NOT NULL DEFAULT 'note',
583 description TEXT NOT NULL DEFAULT '',
584 body TEXT NOT NULL DEFAULT '',
585 body_hash TEXT NOT NULL DEFAULT '',
586 session_id TEXT,
587 source TEXT NOT NULL DEFAULT 'agent',
588 metadata TEXT NOT NULL DEFAULT '{}',
589 created_at INTEGER NOT NULL DEFAULT (unixepoch()),
590 updated_at INTEGER NOT NULL DEFAULT (unixepoch()),
591 deleted_at INTEGER,
592 UNIQUE(namespace, name)
593 );",
594 )
595 .expect("schema creation must succeed");
596 conn
597 }
598
599 fn open_temp_queue() -> (Connection, String) {
600 let path = format!(
601 "/tmp/test-enrich-dl-{}-{}.sqlite",
602 std::process::id(),
603 fastrand::u64(..)
604 );
605 let conn = open_queue_db(&path).expect("queue db must open");
606 (conn, path)
607 }
608
609 fn insert_pending(conn: &Connection, key: &str) -> i64 {
610 conn.execute(
611 "INSERT INTO queue (item_key, item_type, status) VALUES (?1, 'memory', 'pending')",
612 rusqlite::params![key],
613 )
614 .unwrap();
615 conn.last_insert_rowid()
616 }
617
618 #[test]
619 fn queue_db_schema_creates_correctly() {
620 let tmp_path = format!("/tmp/test-enrich-queue-{}.sqlite", std::process::id());
621 let conn = open_queue_db(&tmp_path).expect("queue db must open");
622 let count: i64 = conn
623 .query_row("SELECT COUNT(*) FROM queue", [], |r| r.get(0))
624 .unwrap();
625 assert_eq!(count, 0);
626 let _ = std::fs::remove_file(&tmp_path);
627 }
628
629 #[test]
630 fn classify_rate_limit_is_transient() {
631 let e = AppError::RateLimited {
632 detail: "429".into(),
633 };
634 assert_eq!(
635 classify_enrich_outcome(&e),
636 crate::retry::AttemptOutcome::Transient
637 );
638 }
639
640 #[test]
641 fn classify_timeout_and_dbbusy_are_transient() {
642 let t = AppError::Timeout {
643 operation: "judge".into(),
644 duration_secs: 30,
645 };
646 let b = AppError::DbBusy("locked".into());
647 assert_eq!(
648 classify_enrich_outcome(&t),
649 crate::retry::AttemptOutcome::Transient
650 );
651 assert_eq!(
652 classify_enrich_outcome(&b),
653 crate::retry::AttemptOutcome::Transient
654 );
655 }
656
657 #[test]
658 fn classify_validation_and_parse_are_hard_failure() {
659 let v = AppError::Validation("failed to parse entities array: bad".into());
660 assert_eq!(
661 classify_enrich_outcome(&v),
662 crate::retry::AttemptOutcome::HardFailure
663 );
664 }
665
666 #[test]
667 fn open_queue_db_alter_is_idempotent() {
668 let path = format!(
669 "/tmp/test-enrich-idem-{}-{}.sqlite",
670 std::process::id(),
671 fastrand::u64(..)
672 );
673 let _ = open_queue_db(&path).expect("first open");
674 let conn = open_queue_db(&path).expect("second open is idempotent");
675 let cols: Vec<String> = {
676 let mut stmt = conn.prepare("PRAGMA table_info(queue)").unwrap();
677 stmt.query_map([], |r| r.get::<_, String>(1))
678 .unwrap()
679 .collect::<Result<Vec<_>, _>>()
680 .unwrap()
681 };
682 assert!(cols.iter().any(|c| c == "error_class"));
683 assert!(cols.iter().any(|c| c == "next_retry_at"));
684 let _ = std::fs::remove_file(&path);
685 }
686
687 #[test]
688 fn record_item_failure_hard_marks_dead() {
689 let (conn, path) = open_temp_queue();
690 let id = insert_pending(&conn, "mem-hard");
691 let outcome = record_item_failure(
692 &conn,
693 id,
694 1,
695 5,
696 &AppError::Validation("invalid body".into()),
697 );
698 assert_eq!(outcome, crate::retry::AttemptOutcome::HardFailure);
699 let status: String = conn
700 .query_row(
701 "SELECT status FROM queue WHERE id=?1",
702 rusqlite::params![id],
703 |r| r.get(0),
704 )
705 .unwrap();
706 assert_eq!(status, "dead");
707 let _ = std::fs::remove_file(&path);
708 }
709
710 #[test]
711 fn record_item_failure_transient_reschedules_pending() {
712 let (conn, path) = open_temp_queue();
713 let id = insert_pending(&conn, "mem-transient");
714 let outcome = record_item_failure(
715 &conn,
716 id,
717 1,
718 5,
719 &AppError::RateLimited {
720 detail: "429".into(),
721 },
722 );
723 assert_eq!(outcome, crate::retry::AttemptOutcome::Transient);
724 let (status, future): (String, i64) = conn
725 .query_row(
726 "SELECT status, (next_retry_at > datetime('now')) FROM queue WHERE id=?1",
727 rusqlite::params![id],
728 |r| Ok((r.get(0)?, r.get(1)?)),
729 )
730 .unwrap();
731 assert_eq!(status, "pending");
732 assert_eq!(future, 1, "next_retry_at must be in the future");
733 let _ = std::fs::remove_file(&path);
734 }
735
736 #[test]
737 fn record_item_failure_transient_at_cap_marks_dead() {
738 let (conn, path) = open_temp_queue();
739 let id = insert_pending(&conn, "mem-cap");
740 let outcome = record_item_failure(
741 &conn,
742 id,
743 5,
744 5,
745 &AppError::RateLimited {
746 detail: "429".into(),
747 },
748 );
749 assert_eq!(outcome, crate::retry::AttemptOutcome::Transient);
750 let status: String = conn
751 .query_row(
752 "SELECT status FROM queue WHERE id=?1",
753 rusqlite::params![id],
754 |r| r.get(0),
755 )
756 .unwrap();
757 assert_eq!(status, "dead");
758 let _ = std::fs::remove_file(&path);
759 }
760
761 #[test]
762 fn dequeue_skips_future_retry_and_dead() {
763 let (conn, path) = open_temp_queue();
764 let eligible = insert_pending(&conn, "mem-eligible");
765 let waiting = insert_pending(&conn, "mem-waiting");
766 conn.execute(
767 "UPDATE queue SET next_retry_at=datetime('now', '+3600 seconds') WHERE id=?1",
768 rusqlite::params![waiting],
769 )
770 .unwrap();
771 let dead = insert_pending(&conn, "mem-dead");
772 conn.execute(
773 "UPDATE queue SET status='dead' WHERE id=?1",
774 rusqlite::params![dead],
775 )
776 .unwrap();
777
778 let claimed: Option<i64> = conn
779 .query_row(
780 "UPDATE queue SET status='processing', attempt=attempt+1 \
781 WHERE id = (SELECT id FROM queue WHERE status='pending' \
782 AND (next_retry_at IS NULL OR next_retry_at <= datetime('now')) \
783 ORDER BY id LIMIT 1) \
784 RETURNING id",
785 [],
786 |r| r.get(0),
787 )
788 .ok();
789 assert_eq!(claimed, Some(eligible));
790
791 let second: Option<i64> = conn
792 .query_row(
793 "UPDATE queue SET status='processing', attempt=attempt+1 \
794 WHERE id = (SELECT id FROM queue WHERE status='pending' \
795 AND (next_retry_at IS NULL OR next_retry_at <= datetime('now')) \
796 ORDER BY id LIMIT 1) \
797 RETURNING id",
798 [],
799 |r| r.get(0),
800 )
801 .ok();
802 assert_eq!(second, None);
803 let _ = std::fs::remove_file(&path);
804 }
805
806 #[test]
807 fn classify_validation_never_infers_transience_from_message() {
808 for msg in [
816 "model 'x' returned non-object JSON after repair (got string)",
817 "model 'x' returned content that could not be parsed even after JSON repair",
818 "model 'x' returned no structured content",
819 "LLM result missing 'description' field",
820 "LLM result missing 'enriched_body' field",
821 ] {
822 assert_eq!(
823 classify_enrich_outcome(&AppError::Validation(msg.into())),
824 crate::retry::AttemptOutcome::HardFailure,
825 "expected hard failure for: {msg}"
826 );
827 }
828 }
829
830 #[test]
831 fn classify_embedding_error_is_transient_floor() {
832 assert_eq!(
833 classify_enrich_outcome(&AppError::Embedding("dimension mismatch".into())),
834 crate::retry::AttemptOutcome::Transient
835 );
836 }
837
838 #[test]
841 fn classify_entity_not_yet_materialized_is_transient() {
842 assert_eq!(
843 classify_enrich_outcome(&AppError::EntityNotYetMaterialized {
844 name: "acme".into(),
845 namespace: "global".into(),
846 }),
847 crate::retry::AttemptOutcome::Transient
848 );
849 }
850
851 #[test]
852 fn classify_memory_absence_stays_hard_failure() {
853 assert_eq!(
854 classify_enrich_outcome(&AppError::MemoryNotFound {
855 name: "mem-x".into(),
856 namespace: "global".into(),
857 }),
858 crate::retry::AttemptOutcome::HardFailure
859 );
860 assert_eq!(
861 classify_enrich_outcome(&AppError::MemoryNotFoundById { id: 42 }),
862 crate::retry::AttemptOutcome::HardFailure
863 );
864 assert_eq!(
865 classify_enrich_outcome(&AppError::NotFound("gone".into())),
866 crate::retry::AttemptOutcome::HardFailure
867 );
868 }
869
870 #[test]
871 fn classify_database_busy_is_transient_non_busy_is_hard() {
872 let busy = AppError::Database(rusqlite::Error::SqliteFailure(
873 rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_BUSY),
874 Some("database is locked".into()),
875 ));
876 assert_eq!(
877 classify_enrich_outcome(&busy),
878 crate::retry::AttemptOutcome::Transient
879 );
880 let constraint = AppError::Database(rusqlite::Error::SqliteFailure(
881 rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_CONSTRAINT),
882 Some("UNIQUE constraint failed".into()),
883 ));
884 assert_eq!(
885 classify_enrich_outcome(&constraint),
886 crate::retry::AttemptOutcome::HardFailure
887 );
888 }
889
890 #[test]
891 fn record_item_failure_typed_persists_diagnostics_on_dead_letter() {
892 let (conn, path) = open_temp_queue();
893 let id = insert_pending(&conn, "mem-diag");
894 let outcome = record_item_failure_typed(
895 &conn,
896 id,
897 1,
898 5,
899 crate::retry::AttemptOutcome::HardFailure,
900 "truncated response",
901 Some("length"),
902 Some(120),
903 Some(4096),
904 );
905 assert_eq!(outcome, crate::retry::AttemptOutcome::HardFailure);
906 let (status, finish_reason, input_tokens, output_tokens): (
907 String,
908 Option<String>,
909 Option<i64>,
910 Option<i64>,
911 ) = conn
912 .query_row(
913 "SELECT status, finish_reason, input_tokens, output_tokens FROM queue WHERE id=?1",
914 rusqlite::params![id],
915 |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
916 )
917 .unwrap();
918 assert_eq!(status, "dead");
919 assert_eq!(finish_reason.as_deref(), Some("length"));
920 assert_eq!(input_tokens, Some(120));
921 assert_eq!(output_tokens, Some(4096));
922 let _ = std::fs::remove_file(&path);
923 }
924
925 #[test]
926 fn record_item_failure_typed_reschedules_transient_below_max_attempts() {
927 let (conn, path) = open_temp_queue();
933 let id = insert_pending(&conn, "mem-retry");
934 let outcome = record_item_failure_typed(
935 &conn,
936 id,
937 1,
938 5,
939 crate::retry::AttemptOutcome::Transient,
940 "truncated response",
941 Some("length"),
942 Some(120),
943 Some(64),
944 );
945 assert_eq!(outcome, crate::retry::AttemptOutcome::Transient);
946 let (status, error_class, finish_reason, next_retry_at): (
947 String,
948 String,
949 Option<String>,
950 Option<String>,
951 ) = conn
952 .query_row(
953 "SELECT status, error_class, finish_reason, next_retry_at FROM queue WHERE id=?1",
954 rusqlite::params![id],
955 |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
956 )
957 .unwrap();
958 assert_eq!(status, "pending");
959 assert_eq!(error_class, "transient");
960 assert_eq!(finish_reason.as_deref(), Some("length"));
961 assert!(
962 next_retry_at.is_some(),
963 "a rescheduled item must carry a next_retry_at"
964 );
965 let _ = std::fs::remove_file(&path);
966 }
967
968 #[test]
978 fn with_busy_retry_bounds_dequeue_under_sustained_contention() {
979 let (conn, path) = open_temp_queue();
980 insert_pending(&conn, "mem-busy");
981 conn.pragma_update(None, "busy_timeout", 0i64)
982 .expect("busy_timeout override must succeed");
983
984 let blocker = Connection::open(&path).expect("blocker connection must open");
988 blocker
989 .execute_batch("BEGIN EXCLUSIVE;")
990 .expect("exclusive lock must be acquired");
991
992 let calls = std::sync::Arc::new(std::sync::atomic::AtomicU32::new(0));
993 let calls_clone = std::sync::Arc::clone(&calls);
994 let result: Result<DequeueOutcome, AppError> =
995 crate::storage::utils::with_busy_retry(|| {
996 calls_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
997 dequeue_next_pending(&conn, "")
998 });
999
1000 assert!(
1001 matches!(result, Err(AppError::DbBusy(_))),
1002 "sustained SQLITE_BUSY must convert to DbBusy, not hang or silently report Empty"
1003 );
1004 assert_eq!(
1005 calls.load(std::sync::atomic::Ordering::SeqCst),
1006 crate::constants::MAX_SQLITE_BUSY_RETRIES,
1007 "must attempt exactly MAX_SQLITE_BUSY_RETRIES times, never retry unbounded"
1008 );
1009
1010 blocker
1011 .execute_batch("ROLLBACK;")
1012 .expect("releasing the exclusive lock must succeed");
1013 let _ = std::fs::remove_file(&path);
1014 }
1015
1016 #[test]
1017 fn dequeue_next_pending_distinguishes_empty_from_claimed() {
1018 let (conn, path) = open_temp_queue();
1019 let id = insert_pending(&conn, "mem-dequeue");
1020 let claimed = dequeue_next_pending(&conn, "").expect("dequeue must succeed");
1021 match claimed {
1022 DequeueOutcome::Claimed((claimed_id, key, _, _)) => {
1023 assert_eq!(claimed_id, id);
1024 assert_eq!(key, "mem-dequeue");
1025 }
1026 DequeueOutcome::Empty => panic!("expected a claimed row"),
1027 }
1028 let empty = dequeue_next_pending(&conn, "").expect("dequeue must succeed");
1029 assert!(matches!(empty, DequeueOutcome::Empty));
1030 let _ = std::fs::remove_file(&path);
1031 }
1032
1033 #[test]
1034 fn classify_provider_error_and_not_found_are_hard() {
1035 assert_eq!(
1036 classify_enrich_outcome(&AppError::ProviderError {
1037 code: "400".into(),
1038 message: "context length exceeded".into(),
1039 }),
1040 crate::retry::AttemptOutcome::HardFailure
1041 );
1042 assert_eq!(
1043 classify_enrich_outcome(&AppError::NotFound("memory 'gone' not found".into())),
1044 crate::retry::AttemptOutcome::HardFailure
1045 );
1046 }
1047
1048 #[test]
1049 fn open_queue_db_migrates_operation_column() {
1050 let (conn, path) = open_temp_queue();
1051 drop(conn);
1052 let conn = open_queue_db(&path).expect("second open is idempotent");
1053 let cols: Vec<String> = {
1054 let mut stmt = conn.prepare("PRAGMA table_info(queue)").unwrap();
1055 stmt.query_map([], |r| r.get::<_, String>(1))
1056 .unwrap()
1057 .collect::<Result<Vec<_>, _>>()
1058 .unwrap()
1059 };
1060 assert!(cols.iter().any(|c| c == "operation"));
1061 assert!(cols.iter().any(|c| c == "memory_id"));
1062 let _ = std::fs::remove_file(&path);
1063 }
1064
1065 #[test]
1066 fn enqueue_candidate_tags_operation_and_memory_id() {
1067 let main = open_test_db();
1068 main.execute(
1069 "INSERT INTO memories (namespace, name, body) VALUES ('global', 'mem-x', 'body')",
1070 [],
1071 )
1072 .unwrap();
1073 let mem_id: i64 = main
1074 .query_row("SELECT id FROM memories WHERE name='mem-x'", [], |r| {
1075 r.get(0)
1076 })
1077 .unwrap();
1078 let (queue, path) = open_temp_queue();
1079 enqueue_candidate(&queue, &main, "global", "mem-x", "memory", "MemoryBindings");
1080 let (op, mid): (String, i64) = queue
1081 .query_row(
1082 "SELECT operation, memory_id FROM queue WHERE item_key='mem-x'",
1083 [],
1084 |r| Ok((r.get(0)?, r.get(1)?)),
1085 )
1086 .unwrap();
1087 assert_eq!(op, "MemoryBindings");
1088 assert_eq!(mid, mem_id);
1089 let _ = std::fs::remove_file(&path);
1090 }
1091
1092 #[test]
1093 fn requeue_dead_resurrects_dead_rows() {
1094 let (conn, path) = open_temp_queue();
1095 conn.execute(
1096 "INSERT INTO queue (item_key, item_type, status, operation, attempt, error, error_class, next_retry_at) \
1097 VALUES ('mem-dead', 'memory', 'dead', 'MemoryBindings', 8, 'boom', 'permanent', datetime('now'))",
1098 [],
1099 )
1100 .unwrap();
1101 let n = conn
1102 .execute(
1103 "UPDATE queue SET status='pending', attempt=0, next_retry_at=NULL, \
1104 error=NULL, error_class=NULL \
1105 WHERE status='dead' AND (operation = ?1 OR operation IS NULL)",
1106 rusqlite::params!["MemoryBindings"],
1107 )
1108 .unwrap();
1109 assert_eq!(n, 1);
1110 let (status, attempt, nra): (String, i64, Option<String>) = conn
1111 .query_row(
1112 "SELECT status, attempt, next_retry_at FROM queue WHERE item_key='mem-dead'",
1113 [],
1114 |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
1115 )
1116 .unwrap();
1117 assert_eq!(status, "pending");
1118 assert_eq!(attempt, 0);
1119 assert!(nra.is_none());
1120 let _ = std::fs::remove_file(&path);
1121 }
1122
1123 #[test]
1124 fn skipped_item_keys_excludes_only_skipped_for_operation() {
1125 let (conn, path) = open_temp_queue();
1130 conn.execute(
1131 "INSERT INTO queue (item_key, item_type, status, operation) VALUES ('mem-vetoed', 'memory', 'skipped', 'BodyEnrich')",
1132 [],
1133 )
1134 .unwrap();
1135 conn.execute(
1136 "INSERT INTO queue (item_key, item_type, status, operation) VALUES ('mem-pending', 'memory', 'pending', 'BodyEnrich')",
1137 [],
1138 )
1139 .unwrap();
1140 conn.execute(
1141 "INSERT INTO queue (item_key, item_type, status, operation) VALUES ('mem-other-op', 'memory', 'skipped', 'MemoryBindings')",
1142 [],
1143 )
1144 .unwrap();
1145 let keys = skipped_item_keys(&conn, "BodyEnrich").unwrap();
1146 assert!(
1147 keys.contains("mem-vetoed"),
1148 "vetoed BodyEnrich item must be excluded from scan"
1149 );
1150 assert!(
1151 !keys.contains("mem-pending"),
1152 "pending item is still actionable"
1153 );
1154 assert!(
1155 !keys.contains("mem-other-op"),
1156 "skipped item from another operation must not leak"
1157 );
1158 assert_eq!(keys.len(), 1);
1159 let _ = std::fs::remove_file(&path);
1160 }
1161
1162 #[test]
1163 fn cascade_cleanup_delete_targets_memory_id_and_name() {
1164 let (conn, path) = open_temp_queue();
1165 conn.execute(
1166 "INSERT INTO queue (item_key, item_type, status, memory_id) VALUES ('by-id', 'memory', 'done', 42)",
1167 [],
1168 )
1169 .unwrap();
1170 conn.execute(
1171 "INSERT INTO queue (item_key, item_type, status) VALUES ('by-name', 'memory', 'pending')",
1172 [],
1173 )
1174 .unwrap();
1175 let removed = conn
1176 .execute(
1177 "DELETE FROM queue WHERE memory_id = ?1 OR item_key = ?2",
1178 rusqlite::params![42_i64, "by-name"],
1179 )
1180 .unwrap();
1181 assert_eq!(removed, 2);
1182 let remaining: i64 = conn
1183 .query_row("SELECT COUNT(*) FROM queue", [], |r| r.get(0))
1184 .unwrap();
1185 assert_eq!(remaining, 0);
1186 let _ = std::fs::remove_file(&path);
1187 }
1188
1189 #[test]
1190 fn item_type_for_maps_entity_and_memory() {
1191 assert_eq!(
1192 item_type_for(&EnrichOperation::EntityDescriptions),
1193 "entity"
1194 );
1195 assert_eq!(item_type_for(&EnrichOperation::MemoryBindings), "memory");
1196 assert_eq!(item_type_for(&EnrichOperation::AugmentBindings), "memory");
1197 assert_eq!(item_type_for(&EnrichOperation::BodyExtract), "memory");
1198 }
1199
1200 #[test]
1203 fn item_type_for_key_honours_reembed_prefixes() {
1204 assert_eq!(item_type_for_key("plain-memory-name", "memory"), "memory");
1205 assert_eq!(
1206 item_type_for_key("entity:tokio-runtime", "memory"),
1207 "entity"
1208 );
1209 assert_eq!(item_type_for_key("chunk:42", "memory"), "chunk");
1210 assert_eq!(item_type_for_key("some-entity", "entity"), "entity");
1211 }
1212
1213 #[test]
1214 fn prune_dead_orphans_removes_only_orphan_memory_rows() {
1215 let main = open_test_db();
1216 main.execute(
1218 "INSERT INTO memories (namespace, name, body) VALUES ('global', 'alive', 'b')",
1219 [],
1220 )
1221 .unwrap();
1222 let (queue, path) = open_temp_queue();
1223 queue
1225 .execute(
1226 "INSERT INTO queue (item_key, item_type, status, operation, error_class) \
1227 VALUES ('gone', 'memory', 'dead', 'MemoryBindings', 'permanent')",
1228 [],
1229 )
1230 .unwrap();
1231 queue
1233 .execute(
1234 "INSERT INTO queue (item_key, item_type, status, operation, error_class) \
1235 VALUES ('alive', 'memory', 'dead', 'MemoryBindings', 'permanent')",
1236 [],
1237 )
1238 .unwrap();
1239 queue
1241 .execute(
1242 "INSERT INTO queue (item_key, item_type, status, operation) \
1243 VALUES ('some-entity', 'entity', 'dead', 'EntityDescriptions')",
1244 [],
1245 )
1246 .unwrap();
1247
1248 let pruned = prune_dead_orphans(&queue, &main, "MemoryBindings", "global").unwrap();
1249 assert_eq!(pruned, 1, "only the orphan memory row is pruned");
1250
1251 let remaining: Vec<String> = {
1252 let mut stmt = queue
1253 .prepare("SELECT item_key FROM queue ORDER BY item_key")
1254 .unwrap();
1255 stmt.query_map([], |r| r.get::<_, String>(0))
1256 .unwrap()
1257 .collect::<Result<Vec<_>, _>>()
1258 .unwrap()
1259 };
1260 assert_eq!(remaining, vec!["alive", "some-entity"]);
1261 let _ = std::fs::remove_file(&path);
1262 }
1263
1264 #[test]
1265 fn prune_dead_entity_orphans_removes_only_entity_dead_rows() {
1266 let (queue, path) = open_temp_queue();
1267 queue
1269 .execute(
1270 "INSERT INTO queue (item_key, item_type, status, operation, error_class) \
1271 VALUES ('entity:foo', 'entity', 'dead', 'ReEmbed', 'permanent')",
1272 [],
1273 )
1274 .unwrap();
1275 queue
1277 .execute(
1278 "INSERT INTO queue (item_key, item_type, status, operation, error_class) \
1279 VALUES ('mem-dead', 'memory', 'dead', 'MemoryBindings', 'permanent')",
1280 [],
1281 )
1282 .unwrap();
1283 queue
1285 .execute(
1286 "INSERT INTO queue (item_key, item_type, status, operation) \
1287 VALUES ('entity:bar', 'entity', 'pending', 'ReEmbed')",
1288 [],
1289 )
1290 .unwrap();
1291
1292 let pruned = prune_dead_entity_orphans(&queue, "ReEmbed").unwrap();
1293 assert_eq!(pruned, 1, "only the entity dead row is pruned");
1294
1295 let remaining: Vec<String> = {
1296 let mut stmt = queue
1297 .prepare("SELECT item_key FROM queue ORDER BY item_key")
1298 .unwrap();
1299 stmt.query_map([], |r| r.get::<_, String>(0))
1300 .unwrap()
1301 .collect::<Result<Vec<_>, _>>()
1302 .unwrap()
1303 };
1304 assert_eq!(remaining, vec!["entity:bar", "mem-dead"]);
1305 let _ = std::fs::remove_file(&path);
1306 }
1307}