Skip to main content

sqlite_graphrag/commands/enrich/
queue.rs

1//! Enrichment queue — SQLite-backed scan/retry/dead-letter DB.
2
3use super::*;
4
5// ---------------------------------------------------------------------------
6// Queue DB
7// ---------------------------------------------------------------------------
8
9/// Opens or creates the enrichment queue database.
10///
11/// The queue schema mirrors `ingest_claude` for resume/retry parity.
12/// Uses a different filename (`.enrich-queue.sqlite`) to avoid collision.
13///
14/// # DRY note
15///
16/// This is a near-verbatim copy of `open_queue_db` in `ingest_claude.rs`.
17/// Both should be unified in a shared `llm_runner.rs` module by the
18/// Integration stream.
19pub(super) fn open_queue_db<P: AsRef<std::path::Path>>(path: P) -> Result<Connection, AppError> {
20    let conn = Connection::open(path)?;
21    conn.pragma_update(None, "journal_mode", "wal")?;
22    // GAP-SG-76: without an explicit busy_timeout, a lock contention window
23    // between the dequeue claim and a concurrent worker/main-DB writer
24    // surfaces as SQLITE_BUSY immediately instead of retrying briefly.
25    // Reuses the project-wide canonical value (see rules_rust_sqlite.md —
26    // "DEFINIR busy_timeout em milissegundos explícitos por conexão").
27    conn.pragma_update(None, "busy_timeout", crate::constants::BUSY_TIMEOUT_MILLIS)?;
28    conn.execute_batch(
29        "CREATE TABLE IF NOT EXISTS queue (
30            id          INTEGER PRIMARY KEY AUTOINCREMENT,
31            item_key    TEXT NOT NULL UNIQUE,
32            item_type   TEXT NOT NULL DEFAULT 'memory',
33            status      TEXT NOT NULL DEFAULT 'pending',
34            memory_id   INTEGER,
35            entity_id   INTEGER,
36            entities    INTEGER DEFAULT 0,
37            rels        INTEGER DEFAULT 0,
38            error       TEXT,
39            cost_usd    REAL DEFAULT 0.0,
40            attempt     INTEGER DEFAULT 0,
41            elapsed_ms  INTEGER,
42            created_at  TEXT DEFAULT (datetime('now')),
43            done_at     TEXT
44        );
45        CREATE INDEX IF NOT EXISTS idx_enrich_queue_status ON queue(status);",
46    )?;
47    // GAP-ENRICH-BACKLOG-CONVERGE (v1.0.96): dead-letter columns. The legacy
48    // `.enrich-queue.sqlite` predates these columns and `CREATE TABLE IF NOT
49    // EXISTS` never alters an existing table, so add them idempotently here.
50    let mut has_error_class = false;
51    let mut has_next_retry_at = false;
52    // GAP-SG-12/42: the `operation` column scopes queue rows to the enrich
53    // operation that enqueued them, so `--status` can segment counts per
54    // operation instead of conflating a shared `item_key` space. Migrated
55    // idempotently here for the same reason as the v1.0.96 columns.
56    let mut has_operation = false;
57    // GAP-SG-72: dead-letter diagnostics carried from a typed OpenRouter
58    // `ChatError` (finish_reason + token counts) so `--list-dead` can show
59    // WHY an item died (e.g. truncated by max_tokens) instead of only the
60    // formatted error string. Migrated idempotently for the same reason as
61    // the columns above.
62    let mut has_finish_reason = false;
63    let mut has_input_tokens = false;
64    let mut has_output_tokens = false;
65    {
66        let mut stmt = conn.prepare("PRAGMA table_info(queue)")?;
67        let names = stmt.query_map([], |r| r.get::<_, String>(1))?;
68        for name in names {
69            match name?.as_str() {
70                "error_class" => has_error_class = true,
71                "next_retry_at" => has_next_retry_at = true,
72                "operation" => has_operation = true,
73                "finish_reason" => has_finish_reason = true,
74                "input_tokens" => has_input_tokens = true,
75                "output_tokens" => has_output_tokens = true,
76                _ => {}
77            }
78        }
79    }
80    if !has_error_class {
81        conn.execute_batch("ALTER TABLE queue ADD COLUMN error_class TEXT")?;
82    }
83    if !has_next_retry_at {
84        conn.execute_batch("ALTER TABLE queue ADD COLUMN next_retry_at TEXT")?;
85    }
86    if !has_operation {
87        conn.execute_batch("ALTER TABLE queue ADD COLUMN operation TEXT")?;
88    }
89    if !has_finish_reason {
90        conn.execute_batch("ALTER TABLE queue ADD COLUMN finish_reason TEXT")?;
91    }
92    if !has_input_tokens {
93        conn.execute_batch("ALTER TABLE queue ADD COLUMN input_tokens INTEGER")?;
94    }
95    if !has_output_tokens {
96        conn.execute_batch("ALTER TABLE queue ADD COLUMN output_tokens INTEGER")?;
97    }
98    conn.execute_batch(
99        "CREATE INDEX IF NOT EXISTS idx_enrich_queue_eligible ON queue(status, next_retry_at);
100         CREATE INDEX IF NOT EXISTS idx_enrich_queue_operation ON queue(operation, status);
101         CREATE INDEX IF NOT EXISTS idx_enrich_queue_memory ON queue(memory_id)",
102    )?;
103    Ok(conn)
104}
105
106/// GAP-SG-12: enqueue one scan candidate, linking it to its `memory_id` and
107/// tagging it with the originating `operation`. For memory-keyed operations the
108/// id is resolved from `main_conn` so the cascade cleanup (GAP-SG-13) can target
109/// the queue row by `memory_id` even before the item is processed. Entity/id
110/// keyed operations leave `memory_id` NULL (the `item_key` carries the link).
111/// `INSERT OR IGNORE` preserves the v1.0.96 invariant that a dead-letter row is
112/// never resurrected by re-enqueue (item_key is UNIQUE).
113pub(super) fn enqueue_candidate(
114    queue_conn: &Connection,
115    main_conn: &Connection,
116    namespace: &str,
117    key: &str,
118    item_type: &str,
119    operation: &str,
120) {
121    let memory_id: Option<i64> = if item_type == "memory" {
122        main_conn
123            .query_row(
124                "SELECT id FROM memories WHERE namespace=?1 AND name=?2 AND deleted_at IS NULL",
125                rusqlite::params![namespace, key],
126                |r| r.get(0),
127            )
128            .ok()
129    } else {
130        None
131    };
132    if let Err(e) = queue_conn.execute(
133        "INSERT OR IGNORE INTO queue (item_key, item_type, status, operation, memory_id) \
134         VALUES (?1, ?2, 'pending', ?3, ?4)",
135        rusqlite::params![key, item_type, operation, memory_id],
136    ) {
137        tracing::warn!(target: "enrich", error = %e, "queue insert failed");
138    }
139}
140
141/// GAP-SG-69: item_keys vetoed `status='skipped'` for an operation. The
142/// body-enrich scan selects candidates purely by `LENGTH(body) <
143/// min_output_chars`, so a short body whose rewrite the preservation guard keeps
144/// rejecting would be re-scanned every pass and `--until-empty` would never
145/// converge. Callers exclude these keys so the scan returns only actionable
146/// items; `cleanup_queue_entry` clears the veto when the body actually changes,
147/// restoring the memory as a candidate.
148pub(super) fn skipped_item_keys(
149    conn: &Connection,
150    operation: &str,
151) -> Result<std::collections::HashSet<String>, AppError> {
152    let mut stmt = conn.prepare(
153        "SELECT item_key FROM queue WHERE status='skipped' AND (operation = ?1 OR operation IS NULL)",
154    )?;
155    let keys = stmt
156        .query_map(rusqlite::params![operation], |r| r.get::<_, String>(0))?
157        .collect::<Result<std::collections::HashSet<String>, _>>()?;
158    Ok(keys)
159}
160
161/// Queue `item_type` for an operation: entity-keyed operations use `"entity"`,
162/// every other (memory/id-keyed) operation uses `"memory"`.
163pub(super) fn item_type_for(operation: &EnrichOperation) -> &'static str {
164    match operation {
165        EnrichOperation::EntityDescriptions => "entity",
166        _ => "memory",
167    }
168}
169
170/// v1.1.1 (P2): per-key `item_type` override for the re-embed targets.
171///
172/// Re-embed keys are prefixed with `entity:` / `chunk:` when `--target`
173/// selects a non-memory table; the queue row must carry the real item type
174/// so `prune_dead_orphans` (which only reaps `item_type='memory'` rows)
175/// never mistakes an entity/chunk key for an orphaned memory name.
176/// Unprefixed keys keep the operation-level default.
177pub(super) fn item_type_for_key(key: &str, default: &'static str) -> &'static str {
178    if key.starts_with("entity:") {
179        "entity"
180    } else if key.starts_with("chunk:") {
181        "chunk"
182    } else {
183        default
184    }
185}
186
187/// GAP-SG-13: remove a memory's enrich-queue entry when the memory is deleted or
188/// force-merged, so the dead-letter / pending sidecar never references a row
189/// that no longer exists. Best-effort and a no-op when the queue file is absent
190/// (the common case after a clean run, which removes it). Targets BOTH
191/// `memory_id` (populated at enqueue for memory ops, GAP-SG-12) and `item_key`
192/// (the memory name) so pending rows enqueued before id resolution are also
193/// cleared. Errors are logged, never propagated — cleanup must not fail the
194/// caller's delete/upsert.
195pub fn cleanup_queue_entry(db_path: &std::path::Path, memory_id: i64, name: &str) {
196    let queue_path = crate::paths::sidecar_path(db_path, ".enrich-queue.sqlite");
197    if !queue_path.exists() {
198        return;
199    }
200    match open_queue_db(&queue_path) {
201        Ok(conn) => {
202            if let Err(e) = conn.execute(
203                "DELETE FROM queue WHERE memory_id = ?1 OR item_key = ?2",
204                rusqlite::params![memory_id, name],
205            ) {
206                tracing::warn!(target: "enrich", error = %e, memory_id, "enrich-queue cleanup failed");
207            }
208        }
209        Err(e) => {
210            tracing::warn!(target: "enrich", error = %e, "enrich-queue cleanup skipped (open failed)");
211        }
212    }
213}
214
215/// GAP-SG-66: prune ORPHAN dead-letter rows — `status='dead'` memory rows whose
216/// `item_key` (the memory name) no longer exists in the main DB for `namespace`.
217///
218/// These are terminal "not found" failures (the memory was renamed/purged after
219/// being enqueued): re-processing them just re-fails with the same not-found
220/// error, so `--requeue-dead` can never recover them and they inflate
221/// `queue_dead` forever. Read-only on the main DB; deletes only the
222/// confirmed-orphan rows from the queue sidecar. Entity-keyed dead rows
223/// (`item_type='entity'`) are left untouched — their key is an entity name, not
224/// a memory name. Returns the number of rows pruned.
225pub(super) fn prune_dead_orphans(
226    queue_conn: &Connection,
227    main_conn: &Connection,
228    operation: &str,
229    namespace: &str,
230) -> Result<i64, AppError> {
231    let dead: Vec<(i64, String)> = {
232        let mut stmt = queue_conn.prepare(
233            "SELECT id, item_key FROM queue \
234             WHERE status='dead' AND item_type='memory' \
235             AND (operation = ?1 OR operation IS NULL) ORDER BY id",
236        )?;
237        let rows = stmt
238            .query_map(rusqlite::params![operation], |r| Ok((r.get(0)?, r.get(1)?)))?
239            .collect::<Result<Vec<_>, _>>()?;
240        rows
241    };
242    let mut pruned = 0_i64;
243    for (id, name) in dead {
244        let exists = main_conn
245            .query_row(
246                "SELECT 1 FROM memories WHERE namespace=?1 AND name=?2 AND deleted_at IS NULL",
247                rusqlite::params![namespace, name],
248                |_| Ok(()),
249            )
250            .is_ok();
251        if !exists {
252            queue_conn.execute("DELETE FROM queue WHERE id=?1", rusqlite::params![id])?;
253            pruned += 1;
254        }
255    }
256    if pruned > 0 {
257        let _ = queue_conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);");
258    }
259    Ok(pruned)
260}
261
262/// v1.1.2: prune dead ENTITY orphan rows — remove every `status='dead'`
263/// `item_type='entity'` row from the queue sidecar. Unlike
264/// [`prune_dead_orphans`], this does NOT consult the main DB: entity dead rows
265/// are terminal artifacts of re-extraction/rename and have no recovery path
266/// (re-running them re-fails the same way). Returns the number of rows pruned.
267pub(super) fn prune_dead_entity_orphans(
268    queue_conn: &Connection,
269    operation: &str,
270) -> Result<i64, AppError> {
271    let pruned = queue_conn.execute(
272        "DELETE FROM queue \
273         WHERE status='dead' AND item_type='entity' \
274         AND (operation = ?1 OR operation IS NULL)",
275        rusqlite::params![operation],
276    )? as i64;
277    if pruned > 0 {
278        let _ = queue_conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);");
279    }
280    Ok(pruned)
281}
282
283// ---------------------------------------------------------------------------
284// GAP-ENRICH-BACKLOG-CONVERGE — dead-letter classification + queue failure sink
285// ---------------------------------------------------------------------------
286
287/// Read-only `enrich --status` report (no LLM, no singleton).
288///
289/// GAP-SG-42: all queue counts are scoped to the current `--operation` (rows
290/// migrated before the `operation` column, which are NULL, are still counted so
291/// a legacy queue is not silently reported as empty).
292#[derive(Debug, Serialize, schemars::JsonSchema)]
293pub struct EnrichStatus {
294    pub(super) status_report: bool,
295    pub(super) operation: String,
296    pub(super) namespace: String,
297    pub(super) unbound_backlog: usize,
298    /// GAP-SG-77: DATABASE-semantics backlog for the queried operation, computed
299    /// by `scan::count_operation_backlog` via a `SELECT COUNT(*)` over the real
300    /// store. This is distinct from `queue_pending`/`queue_dead` (FILE/sidecar
301    /// queue semantics) and from the legacy `unbound_backlog` (memory-bindings
302    /// only). It fixes the false `pending=0` that db-backed operations
303    /// (entity-descriptions/body-enrich/re-embed) previously reported.
304    pub(super) scan_backlog: i64,
305    pub(super) queue_pending: i64,
306    pub(super) queue_processing: i64,
307    pub(super) queue_done: i64,
308    pub(super) queue_failed: i64,
309    pub(super) queue_skipped: i64,
310    pub(super) queue_dead: i64,
311    pub(super) eligible_now: i64,
312    pub(super) waiting: i64,
313    /// GAP-SG-15/46: coarse backlog state, disambiguating an empty queue from a
314    /// not-yet-scanned backlog and from a cooldown wait.
315    /// `draining` (eligible items now) | `cooldown` (all pending items waiting on
316    /// `next_retry_at`) | `pending-scan` (candidates exist but the queue is not
317    /// populated — run enrich to scan) | `empty` (nothing left to do).
318    pub(super) state: &'static str,
319    /// GAP-SG-16: per-item `next_retry_at` for every pending row currently in
320    /// backoff, so an operator can see exactly when each will become eligible.
321    pub(super) waiting_items: Vec<WaitingItem>,
322}
323
324/// GAP-SG-16: one pending queue row waiting on its backoff cooldown.
325#[derive(Debug, Serialize, schemars::JsonSchema)]
326pub struct WaitingItem {
327    pub(super) item_key: String,
328    pub(super) attempt: i64,
329    pub(super) next_retry_at: Option<String>,
330    pub(super) error_class: Option<String>,
331}
332
333/// GAP-SG-23: one dead-letter row reported by `--list-dead`.
334#[derive(Debug, Serialize, schemars::JsonSchema)]
335pub struct DeadItem {
336    pub(super) dead_item: bool,
337    pub(super) item_key: String,
338    pub(super) item_type: String,
339    pub(super) attempt: i64,
340    pub(super) error_class: Option<String>,
341    pub(super) error: Option<String>,
342    /// GAP-SG-72: `choices[0].finish_reason` from the OpenRouter response
343    /// that produced this failure, when one was decoded (e.g. `"length"`
344    /// for a max_tokens truncation). `None` for subprocess-provider modes
345    /// or failures that never reached a decoded response.
346    pub(super) finish_reason: Option<String>,
347    /// GAP-SG-72: `usage.prompt_tokens` from the same response, when known.
348    pub(super) input_tokens: Option<i64>,
349    /// GAP-SG-72: `usage.completion_tokens` from the same response, when known.
350    pub(super) output_tokens: Option<i64>,
351}
352
353/// GAP-SG-23/11: summary footer for `--list-dead` and `--requeue-dead`.
354#[derive(Debug, Serialize, schemars::JsonSchema)]
355pub struct DeadSummary {
356    pub(super) summary: bool,
357    pub(super) operation: String,
358    pub(super) namespace: String,
359    /// `list-dead` | `requeue-dead` | `prune-dead-orphans`
360    pub(super) action: &'static str,
361    pub(super) dead_total: i64,
362    pub(super) requeued: i64,
363    /// GAP-SG-66: `prune-dead-orphans` — dead rows removed because their
364    /// referenced memory no longer exists in the main DB for the namespace.
365    /// Zero for `list-dead` / `requeue-dead`.
366    pub(super) pruned: i64,
367}
368
369/// Classifies an enrich item failure into a retry/dead-letter outcome.
370///
371/// This is the FALLBACK classifier: it is only consulted when the failure
372/// did not already carry a typed [`crate::retry::AttemptOutcome`] computed at
373/// its origin (see [`record_item_failure_typed`], fed by
374/// [`crate::commands::enrich::extraction::take_last_openrouter_failure`] for
375/// OpenRouter chat/embedding calls). Classification is TYPED by `AppError`
376/// variant only — NEVER by matching the formatted message — per
377/// `rules_rust_retry_com_backoff.md` ("NUNCA usar string matching em
378/// mensagens de erro").
379pub(super) fn classify_enrich_outcome(e: &AppError) -> crate::retry::AttemptOutcome {
380    use crate::retry::AttemptOutcome;
381    match e {
382        AppError::RateLimited { .. } | AppError::Timeout { .. } | AppError::DbBusy(_) => {
383            AttemptOutcome::Transient
384        }
385        // GAP-SG-78: a referenced entity that is not yet materialized is a
386        // TRANSITORY absence — a later enrich pass creates the entity — so the
387        // item is rescheduled, not dead-lettered on the first miss. Matched on
388        // the typed variant, never a message substring (rules_rust_retry: NUNCA
389        // string matching). The `--max-attempts` floor (default 8) still ends
390        // the item if the entity never materializes, mirroring the `Embedding`
391        // floor below.
392        AppError::EntityNotYetMaterialized { .. } => AttemptOutcome::Transient,
393        // GAP-SG-09: errors that are genuinely PERMANENT for this item and must
394        // dead-letter immediately (retrying cannot help): a structured provider
395        // rejection (context-length overflow / refusal carried as ProviderError),
396        // or a MEMORY that no longer exists (deleted or renamed between scan and
397        // processing). Entity absence is handled above as transitory, NOT here.
398        AppError::ProviderError { .. }
399        | AppError::NotFound(_)
400        | AppError::MemoryNotFound { .. }
401        | AppError::MemoryNotFoundById { .. } => AttemptOutcome::HardFailure,
402        // GAP-SG-76: SQLITE_BUSY/LOCKED is a lock-contention hiccup between the
403        // queue writer and a concurrent claim — retry it; any other database
404        // error (constraint violation, corruption, I/O) is permanent.
405        AppError::Database(_) => {
406            if crate::storage::utils::is_sqlite_busy(e) {
407                AttemptOutcome::Transient
408            } else {
409                AttemptOutcome::HardFailure
410            }
411        }
412        // GAP-SG-73: safe floor for the `re-embed` operation. `AppError::Embedding`
413        // reaches here only via `embed_with_fallback`'s backend-chain resolution
414        // (`crate::embedder`), which discards the origin-typed
415        // `EmbedError::retry_class` through `From<EmbedError> for AppError` before
416        // the error surfaces to the queue. Extracting the precise verdict would
417        // require bypassing the fallback chain to call the OpenRouter embedding
418        // client directly — out of scope here (touches `embedder.rs`, which is
419        // off-limits, and removes the multi-backend fallback safety net).
420        // Transient is the conservative choice: a persistently permanent failure
421        // still terminates via `--max-attempts` instead of retrying forever.
422        AppError::Embedding(_) => AttemptOutcome::Transient,
423        // Every other variant — including `Validation` without an
424        // origin-typed retry verdict attached — is treated as permanent.
425        // Previously this branch inspected the formatted message for
426        // substrings like "json" / "missing '" to guess at transience; that
427        // guesswork is now unnecessary because the OpenRouter chat path
428        // (the project's only supported enrich mode) attaches its retry
429        // verdict directly via `ChatError::retry_class`, computed at the
430        // exact HTTP status / provider code in `chat_api.rs`, and
431        // `record_item_failure_typed` consumes it BEFORE ever falling back
432        // to this classifier.
433        _ => AttemptOutcome::HardFailure,
434    }
435}
436
437/// Applies a failure outcome to a single queue row. Shared by the parallel
438/// worker and the serial loop (DRY). A `HardFailure`, or a transient failure
439/// whose attempt count reached `max_attempts`, lands in the dead-letter status
440/// (`status='dead'`) so it is never re-selected. A transient failure below the
441/// cap is rescheduled to `pending` with an exponential-backoff `next_retry_at`.
442/// Returns the [`crate::retry::AttemptOutcome`] so the caller can feed the
443/// existing circuit breaker.
444///
445/// GAP-SG-73: delegates to [`record_item_failure_typed`] with the outcome
446/// computed by the untyped fallback classifier and no diagnostics — the
447/// entry point for callers that only have a bare `&AppError` (subprocess
448/// providers, persistence failures).
449pub(super) fn record_item_failure(
450    queue_conn: &rusqlite::Connection,
451    queue_id: i64,
452    attempt: i64,
453    max_attempts: u32,
454    err: &AppError,
455) -> crate::retry::AttemptOutcome {
456    let outcome = classify_enrich_outcome(err);
457    let err_str = format!("{err}");
458    record_item_failure_typed(
459        queue_conn,
460        queue_id,
461        attempt,
462        max_attempts,
463        outcome,
464        &err_str,
465        None,
466        None,
467        None,
468    )
469}
470
471/// GAP-SG-72/73: applies a failure outcome to a single queue row using an
472/// [`crate::retry::AttemptOutcome`] the caller ALREADY computed at the
473/// failure's origin (e.g. `ChatError::retry_class` from an OpenRouter chat
474/// call), plus whatever truncation diagnostics (`finish_reason` and token
475/// counts) were available. This is the precise counterpart to
476/// [`record_item_failure`], which falls back to the untyped
477/// [`classify_enrich_outcome`] classifier when no origin-typed verdict
478/// exists. Both share this single write path (DRY).
479#[allow(clippy::too_many_arguments)]
480pub(super) fn record_item_failure_typed(
481    queue_conn: &rusqlite::Connection,
482    queue_id: i64,
483    attempt: i64,
484    max_attempts: u32,
485    outcome: crate::retry::AttemptOutcome,
486    err_str: &str,
487    finish_reason: Option<&str>,
488    input_tokens: Option<i64>,
489    output_tokens: Option<i64>,
490) -> crate::retry::AttemptOutcome {
491    use crate::retry::AttemptOutcome;
492    let error_class = match outcome {
493        AttemptOutcome::Transient => "transient",
494        AttemptOutcome::HardFailure => "permanent",
495        AttemptOutcome::Success => "success",
496    };
497
498    let terminal = matches!(outcome, AttemptOutcome::HardFailure) || attempt >= max_attempts as i64;
499    if terminal {
500        let _ = queue_conn.execute(
501            "UPDATE queue SET status='dead', error=?1, error_class=?2, done_at=datetime('now'), \
502             finish_reason=?3, input_tokens=?4, output_tokens=?5 WHERE id=?6",
503            rusqlite::params![
504                err_str,
505                error_class,
506                finish_reason,
507                input_tokens,
508                output_tokens,
509                queue_id
510            ],
511        );
512    } else {
513        let delay = crate::retry::compute_delay(
514            &crate::retry::RetryConfig::llm_rate_limit(),
515            attempt.max(0) as u32,
516        );
517        let secs = delay.as_secs().max(1);
518        let modifier = format!("+{secs} seconds");
519        let _ = queue_conn.execute(
520            "UPDATE queue SET status='pending', error=?1, error_class=?2, next_retry_at=datetime('now', ?3), \
521             finish_reason=?4, input_tokens=?5, output_tokens=?6 WHERE id=?7",
522            rusqlite::params![
523                err_str,
524                error_class,
525                modifier,
526                finish_reason,
527                input_tokens,
528                output_tokens,
529                queue_id
530            ],
531        );
532    }
533    outcome
534}
535
536/// GAP-SG-76: outcome of claiming the next pending queue row. Distinguishes
537/// a genuinely empty backlog (`QueryReturnedNoRows`) from lock contention
538/// (`SQLITE_BUSY`/`SQLITE_LOCKED`) so the caller retries briefly on the
539/// latter instead of breaking out of the drain loop early. Both the serial
540/// loop and the parallel worker loop share this (DRY) — previously each
541/// collapsed every `query_row` error into `.ok()`, silently treating a busy
542/// database the same as an empty queue.
543pub(super) enum DequeueOutcome {
544    Claimed((i64, String, String, i64)),
545    Empty,
546}
547
548pub(super) fn dequeue_next_pending(
549    queue_conn: &rusqlite::Connection,
550    backoff_clause: &str,
551) -> Result<DequeueOutcome, AppError> {
552    let dequeue_sql = format!(
553        "UPDATE queue SET status='processing', attempt=attempt+1 \
554         WHERE id = (SELECT id FROM queue WHERE status='pending' {backoff_clause} \
555                     ORDER BY id LIMIT 1) \
556         RETURNING id, item_key, item_type, attempt"
557    );
558    match queue_conn.query_row(&dequeue_sql, [], |row| {
559        Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?))
560    }) {
561        Ok(claimed) => Ok(DequeueOutcome::Claimed(claimed)),
562        Err(rusqlite::Error::QueryReturnedNoRows) => Ok(DequeueOutcome::Empty),
563        Err(e) => Err(AppError::Database(e)),
564    }
565}
566
567// ---------------------------------------------------------------------------
568// Tests
569// ---------------------------------------------------------------------------
570
571#[cfg(test)]
572mod tests {
573    use super::*;
574
575    fn open_test_db() -> Connection {
576        let conn = Connection::open_in_memory().expect("in-memory db");
577        conn.execute_batch(
578            "CREATE TABLE memories (
579                id          INTEGER PRIMARY KEY AUTOINCREMENT,
580                namespace   TEXT NOT NULL DEFAULT 'global',
581                name        TEXT NOT NULL,
582                type        TEXT NOT NULL DEFAULT 'note',
583                description TEXT NOT NULL DEFAULT '',
584                body        TEXT NOT NULL DEFAULT '',
585                body_hash   TEXT NOT NULL DEFAULT '',
586                session_id  TEXT,
587                source      TEXT NOT NULL DEFAULT 'agent',
588                metadata    TEXT NOT NULL DEFAULT '{}',
589                created_at  INTEGER NOT NULL DEFAULT (unixepoch()),
590                updated_at  INTEGER NOT NULL DEFAULT (unixepoch()),
591                deleted_at  INTEGER,
592                UNIQUE(namespace, name)
593            );",
594        )
595        .expect("schema creation must succeed");
596        conn
597    }
598
599    fn open_temp_queue() -> (Connection, String) {
600        let path = format!(
601            "/tmp/test-enrich-dl-{}-{}.sqlite",
602            std::process::id(),
603            fastrand::u64(..)
604        );
605        let conn = open_queue_db(&path).expect("queue db must open");
606        (conn, path)
607    }
608
609    fn insert_pending(conn: &Connection, key: &str) -> i64 {
610        conn.execute(
611            "INSERT INTO queue (item_key, item_type, status) VALUES (?1, 'memory', 'pending')",
612            rusqlite::params![key],
613        )
614        .unwrap();
615        conn.last_insert_rowid()
616    }
617
618    #[test]
619    fn queue_db_schema_creates_correctly() {
620        let tmp_path = format!("/tmp/test-enrich-queue-{}.sqlite", std::process::id());
621        let conn = open_queue_db(&tmp_path).expect("queue db must open");
622        let count: i64 = conn
623            .query_row("SELECT COUNT(*) FROM queue", [], |r| r.get(0))
624            .unwrap();
625        assert_eq!(count, 0);
626        let _ = std::fs::remove_file(&tmp_path);
627    }
628
629    #[test]
630    fn classify_rate_limit_is_transient() {
631        let e = AppError::RateLimited {
632            detail: "429".into(),
633        };
634        assert_eq!(
635            classify_enrich_outcome(&e),
636            crate::retry::AttemptOutcome::Transient
637        );
638    }
639
640    #[test]
641    fn classify_timeout_and_dbbusy_are_transient() {
642        let t = AppError::Timeout {
643            operation: "judge".into(),
644            duration_secs: 30,
645        };
646        let b = AppError::DbBusy("locked".into());
647        assert_eq!(
648            classify_enrich_outcome(&t),
649            crate::retry::AttemptOutcome::Transient
650        );
651        assert_eq!(
652            classify_enrich_outcome(&b),
653            crate::retry::AttemptOutcome::Transient
654        );
655    }
656
657    #[test]
658    fn classify_validation_and_parse_are_hard_failure() {
659        let v = AppError::Validation("failed to parse entities array: bad".into());
660        assert_eq!(
661            classify_enrich_outcome(&v),
662            crate::retry::AttemptOutcome::HardFailure
663        );
664    }
665
666    #[test]
667    fn open_queue_db_alter_is_idempotent() {
668        let path = format!(
669            "/tmp/test-enrich-idem-{}-{}.sqlite",
670            std::process::id(),
671            fastrand::u64(..)
672        );
673        let _ = open_queue_db(&path).expect("first open");
674        let conn = open_queue_db(&path).expect("second open is idempotent");
675        let cols: Vec<String> = {
676            let mut stmt = conn.prepare("PRAGMA table_info(queue)").unwrap();
677            stmt.query_map([], |r| r.get::<_, String>(1))
678                .unwrap()
679                .collect::<Result<Vec<_>, _>>()
680                .unwrap()
681        };
682        assert!(cols.iter().any(|c| c == "error_class"));
683        assert!(cols.iter().any(|c| c == "next_retry_at"));
684        let _ = std::fs::remove_file(&path);
685    }
686
687    #[test]
688    fn record_item_failure_hard_marks_dead() {
689        let (conn, path) = open_temp_queue();
690        let id = insert_pending(&conn, "mem-hard");
691        let outcome = record_item_failure(
692            &conn,
693            id,
694            1,
695            5,
696            &AppError::Validation("invalid body".into()),
697        );
698        assert_eq!(outcome, crate::retry::AttemptOutcome::HardFailure);
699        let status: String = conn
700            .query_row(
701                "SELECT status FROM queue WHERE id=?1",
702                rusqlite::params![id],
703                |r| r.get(0),
704            )
705            .unwrap();
706        assert_eq!(status, "dead");
707        let _ = std::fs::remove_file(&path);
708    }
709
710    #[test]
711    fn record_item_failure_transient_reschedules_pending() {
712        let (conn, path) = open_temp_queue();
713        let id = insert_pending(&conn, "mem-transient");
714        let outcome = record_item_failure(
715            &conn,
716            id,
717            1,
718            5,
719            &AppError::RateLimited {
720                detail: "429".into(),
721            },
722        );
723        assert_eq!(outcome, crate::retry::AttemptOutcome::Transient);
724        let (status, future): (String, i64) = conn
725            .query_row(
726                "SELECT status, (next_retry_at > datetime('now')) FROM queue WHERE id=?1",
727                rusqlite::params![id],
728                |r| Ok((r.get(0)?, r.get(1)?)),
729            )
730            .unwrap();
731        assert_eq!(status, "pending");
732        assert_eq!(future, 1, "next_retry_at must be in the future");
733        let _ = std::fs::remove_file(&path);
734    }
735
736    #[test]
737    fn record_item_failure_transient_at_cap_marks_dead() {
738        let (conn, path) = open_temp_queue();
739        let id = insert_pending(&conn, "mem-cap");
740        let outcome = record_item_failure(
741            &conn,
742            id,
743            5,
744            5,
745            &AppError::RateLimited {
746                detail: "429".into(),
747            },
748        );
749        assert_eq!(outcome, crate::retry::AttemptOutcome::Transient);
750        let status: String = conn
751            .query_row(
752                "SELECT status FROM queue WHERE id=?1",
753                rusqlite::params![id],
754                |r| r.get(0),
755            )
756            .unwrap();
757        assert_eq!(status, "dead");
758        let _ = std::fs::remove_file(&path);
759    }
760
761    #[test]
762    fn dequeue_skips_future_retry_and_dead() {
763        let (conn, path) = open_temp_queue();
764        let eligible = insert_pending(&conn, "mem-eligible");
765        let waiting = insert_pending(&conn, "mem-waiting");
766        conn.execute(
767            "UPDATE queue SET next_retry_at=datetime('now', '+3600 seconds') WHERE id=?1",
768            rusqlite::params![waiting],
769        )
770        .unwrap();
771        let dead = insert_pending(&conn, "mem-dead");
772        conn.execute(
773            "UPDATE queue SET status='dead' WHERE id=?1",
774            rusqlite::params![dead],
775        )
776        .unwrap();
777
778        let claimed: Option<i64> = conn
779            .query_row(
780                "UPDATE queue SET status='processing', attempt=attempt+1 \
781                 WHERE id = (SELECT id FROM queue WHERE status='pending' \
782                               AND (next_retry_at IS NULL OR next_retry_at <= datetime('now')) \
783                             ORDER BY id LIMIT 1) \
784                 RETURNING id",
785                [],
786                |r| r.get(0),
787            )
788            .ok();
789        assert_eq!(claimed, Some(eligible));
790
791        let second: Option<i64> = conn
792            .query_row(
793                "UPDATE queue SET status='processing', attempt=attempt+1 \
794                 WHERE id = (SELECT id FROM queue WHERE status='pending' \
795                               AND (next_retry_at IS NULL OR next_retry_at <= datetime('now')) \
796                             ORDER BY id LIMIT 1) \
797                 RETURNING id",
798                [],
799                |r| r.get(0),
800            )
801            .ok();
802        assert_eq!(second, None);
803        let _ = std::fs::remove_file(&path);
804    }
805
806    #[test]
807    fn classify_validation_never_infers_transience_from_message() {
808        // GAP-SG-73: the fallback classifier is TYPED-only now. Messages
809        // that used to be sniffed for "json" / "missing '" substrings and
810        // treated as Transient are HardFailure here — the OpenRouter chat
811        // path (the project's only supported enrich mode) attaches its own
812        // typed `ChatError::retry_class` for these exact shape failures
813        // BEFORE `record_item_failure_typed` ever falls back to this
814        // classifier, so no message-based guessing survives in the fallback.
815        for msg in [
816            "model 'x' returned non-object JSON after repair (got string)",
817            "model 'x' returned content that could not be parsed even after JSON repair",
818            "model 'x' returned no structured content",
819            "LLM result missing 'description' field",
820            "LLM result missing 'enriched_body' field",
821        ] {
822            assert_eq!(
823                classify_enrich_outcome(&AppError::Validation(msg.into())),
824                crate::retry::AttemptOutcome::HardFailure,
825                "expected hard failure for: {msg}"
826            );
827        }
828    }
829
830    #[test]
831    fn classify_embedding_error_is_transient_floor() {
832        assert_eq!(
833            classify_enrich_outcome(&AppError::Embedding("dimension mismatch".into())),
834            crate::retry::AttemptOutcome::Transient
835        );
836    }
837
838    // GAP-SG-78: entity absence is Transient (own typed variant); memory
839    // absence and the untyped NotFound string stay HardFailure. No substring.
840    #[test]
841    fn classify_entity_not_yet_materialized_is_transient() {
842        assert_eq!(
843            classify_enrich_outcome(&AppError::EntityNotYetMaterialized {
844                name: "acme".into(),
845                namespace: "global".into(),
846            }),
847            crate::retry::AttemptOutcome::Transient
848        );
849    }
850
851    #[test]
852    fn classify_memory_absence_stays_hard_failure() {
853        assert_eq!(
854            classify_enrich_outcome(&AppError::MemoryNotFound {
855                name: "mem-x".into(),
856                namespace: "global".into(),
857            }),
858            crate::retry::AttemptOutcome::HardFailure
859        );
860        assert_eq!(
861            classify_enrich_outcome(&AppError::MemoryNotFoundById { id: 42 }),
862            crate::retry::AttemptOutcome::HardFailure
863        );
864        assert_eq!(
865            classify_enrich_outcome(&AppError::NotFound("gone".into())),
866            crate::retry::AttemptOutcome::HardFailure
867        );
868    }
869
870    #[test]
871    fn classify_database_busy_is_transient_non_busy_is_hard() {
872        let busy = AppError::Database(rusqlite::Error::SqliteFailure(
873            rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_BUSY),
874            Some("database is locked".into()),
875        ));
876        assert_eq!(
877            classify_enrich_outcome(&busy),
878            crate::retry::AttemptOutcome::Transient
879        );
880        let constraint = AppError::Database(rusqlite::Error::SqliteFailure(
881            rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_CONSTRAINT),
882            Some("UNIQUE constraint failed".into()),
883        ));
884        assert_eq!(
885            classify_enrich_outcome(&constraint),
886            crate::retry::AttemptOutcome::HardFailure
887        );
888    }
889
890    #[test]
891    fn record_item_failure_typed_persists_diagnostics_on_dead_letter() {
892        let (conn, path) = open_temp_queue();
893        let id = insert_pending(&conn, "mem-diag");
894        let outcome = record_item_failure_typed(
895            &conn,
896            id,
897            1,
898            5,
899            crate::retry::AttemptOutcome::HardFailure,
900            "truncated response",
901            Some("length"),
902            Some(120),
903            Some(4096),
904        );
905        assert_eq!(outcome, crate::retry::AttemptOutcome::HardFailure);
906        let (status, finish_reason, input_tokens, output_tokens): (
907            String,
908            Option<String>,
909            Option<i64>,
910            Option<i64>,
911        ) = conn
912            .query_row(
913                "SELECT status, finish_reason, input_tokens, output_tokens FROM queue WHERE id=?1",
914                rusqlite::params![id],
915                |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
916            )
917            .unwrap();
918        assert_eq!(status, "dead");
919        assert_eq!(finish_reason.as_deref(), Some("length"));
920        assert_eq!(input_tokens, Some(120));
921        assert_eq!(output_tokens, Some(4096));
922        let _ = std::fs::remove_file(&path);
923    }
924
925    #[test]
926    fn record_item_failure_typed_reschedules_transient_below_max_attempts() {
927        // GAP-SG-72-chat: a transient failure (e.g. a truncated OpenRouter
928        // response) below max_attempts must stay `pending` with a
929        // future `next_retry_at`, not go straight to `dead` — and it must
930        // still persist the finish_reason/token diagnostics for later
931        // inspection via `--list-dead` / `--status`.
932        let (conn, path) = open_temp_queue();
933        let id = insert_pending(&conn, "mem-retry");
934        let outcome = record_item_failure_typed(
935            &conn,
936            id,
937            1,
938            5,
939            crate::retry::AttemptOutcome::Transient,
940            "truncated response",
941            Some("length"),
942            Some(120),
943            Some(64),
944        );
945        assert_eq!(outcome, crate::retry::AttemptOutcome::Transient);
946        let (status, error_class, finish_reason, next_retry_at): (
947            String,
948            String,
949            Option<String>,
950            Option<String>,
951        ) = conn
952            .query_row(
953                "SELECT status, error_class, finish_reason, next_retry_at FROM queue WHERE id=?1",
954                rusqlite::params![id],
955                |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)),
956            )
957            .unwrap();
958        assert_eq!(status, "pending");
959        assert_eq!(error_class, "transient");
960        assert_eq!(finish_reason.as_deref(), Some("length"));
961        assert!(
962            next_retry_at.is_some(),
963            "a rescheduled item must carry a next_retry_at"
964        );
965        let _ = std::fs::remove_file(&path);
966    }
967
968    /// GAP-SG-76/v1.1.00 fix: proves the enrich drain loops' composition
969    /// `with_busy_retry(|| dequeue_next_pending(...))` is BOUNDED under
970    /// sustained lock contention instead of the previous
971    /// `loop { ... continue; }`, which retried `SQLITE_BUSY` forever. A
972    /// second connection holds an exclusive write lock for the whole test;
973    /// the queue connection under test has `busy_timeout=0` so SQLite
974    /// reports `SQLITE_BUSY` immediately instead of blocking internally,
975    /// isolating `with_busy_retry`'s own bounded backoff (5 attempts) as the
976    /// only source of delay.
977    #[test]
978    fn with_busy_retry_bounds_dequeue_under_sustained_contention() {
979        let (conn, path) = open_temp_queue();
980        insert_pending(&conn, "mem-busy");
981        conn.pragma_update(None, "busy_timeout", 0i64)
982            .expect("busy_timeout override must succeed");
983
984        // Second connection holds an EXCLUSIVE write lock so every dequeue
985        // attempt on `conn` observes SQLITE_BUSY, never SQLITE_LOCKED-then-
986        // clears-up.
987        let blocker = Connection::open(&path).expect("blocker connection must open");
988        blocker
989            .execute_batch("BEGIN EXCLUSIVE;")
990            .expect("exclusive lock must be acquired");
991
992        let calls = std::sync::Arc::new(std::sync::atomic::AtomicU32::new(0));
993        let calls_clone = std::sync::Arc::clone(&calls);
994        let result: Result<DequeueOutcome, AppError> =
995            crate::storage::utils::with_busy_retry(|| {
996                calls_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
997                dequeue_next_pending(&conn, "")
998            });
999
1000        assert!(
1001            matches!(result, Err(AppError::DbBusy(_))),
1002            "sustained SQLITE_BUSY must convert to DbBusy, not hang or silently report Empty"
1003        );
1004        assert_eq!(
1005            calls.load(std::sync::atomic::Ordering::SeqCst),
1006            crate::constants::MAX_SQLITE_BUSY_RETRIES,
1007            "must attempt exactly MAX_SQLITE_BUSY_RETRIES times, never retry unbounded"
1008        );
1009
1010        blocker
1011            .execute_batch("ROLLBACK;")
1012            .expect("releasing the exclusive lock must succeed");
1013        let _ = std::fs::remove_file(&path);
1014    }
1015
1016    #[test]
1017    fn dequeue_next_pending_distinguishes_empty_from_claimed() {
1018        let (conn, path) = open_temp_queue();
1019        let id = insert_pending(&conn, "mem-dequeue");
1020        let claimed = dequeue_next_pending(&conn, "").expect("dequeue must succeed");
1021        match claimed {
1022            DequeueOutcome::Claimed((claimed_id, key, _, _)) => {
1023                assert_eq!(claimed_id, id);
1024                assert_eq!(key, "mem-dequeue");
1025            }
1026            DequeueOutcome::Empty => panic!("expected a claimed row"),
1027        }
1028        let empty = dequeue_next_pending(&conn, "").expect("dequeue must succeed");
1029        assert!(matches!(empty, DequeueOutcome::Empty));
1030        let _ = std::fs::remove_file(&path);
1031    }
1032
1033    #[test]
1034    fn classify_provider_error_and_not_found_are_hard() {
1035        assert_eq!(
1036            classify_enrich_outcome(&AppError::ProviderError {
1037                code: "400".into(),
1038                message: "context length exceeded".into(),
1039            }),
1040            crate::retry::AttemptOutcome::HardFailure
1041        );
1042        assert_eq!(
1043            classify_enrich_outcome(&AppError::NotFound("memory 'gone' not found".into())),
1044            crate::retry::AttemptOutcome::HardFailure
1045        );
1046    }
1047
1048    #[test]
1049    fn open_queue_db_migrates_operation_column() {
1050        let (conn, path) = open_temp_queue();
1051        drop(conn);
1052        let conn = open_queue_db(&path).expect("second open is idempotent");
1053        let cols: Vec<String> = {
1054            let mut stmt = conn.prepare("PRAGMA table_info(queue)").unwrap();
1055            stmt.query_map([], |r| r.get::<_, String>(1))
1056                .unwrap()
1057                .collect::<Result<Vec<_>, _>>()
1058                .unwrap()
1059        };
1060        assert!(cols.iter().any(|c| c == "operation"));
1061        assert!(cols.iter().any(|c| c == "memory_id"));
1062        let _ = std::fs::remove_file(&path);
1063    }
1064
1065    #[test]
1066    fn enqueue_candidate_tags_operation_and_memory_id() {
1067        let main = open_test_db();
1068        main.execute(
1069            "INSERT INTO memories (namespace, name, body) VALUES ('global', 'mem-x', 'body')",
1070            [],
1071        )
1072        .unwrap();
1073        let mem_id: i64 = main
1074            .query_row("SELECT id FROM memories WHERE name='mem-x'", [], |r| {
1075                r.get(0)
1076            })
1077            .unwrap();
1078        let (queue, path) = open_temp_queue();
1079        enqueue_candidate(&queue, &main, "global", "mem-x", "memory", "MemoryBindings");
1080        let (op, mid): (String, i64) = queue
1081            .query_row(
1082                "SELECT operation, memory_id FROM queue WHERE item_key='mem-x'",
1083                [],
1084                |r| Ok((r.get(0)?, r.get(1)?)),
1085            )
1086            .unwrap();
1087        assert_eq!(op, "MemoryBindings");
1088        assert_eq!(mid, mem_id);
1089        let _ = std::fs::remove_file(&path);
1090    }
1091
1092    #[test]
1093    fn requeue_dead_resurrects_dead_rows() {
1094        let (conn, path) = open_temp_queue();
1095        conn.execute(
1096            "INSERT INTO queue (item_key, item_type, status, operation, attempt, error, error_class, next_retry_at) \
1097             VALUES ('mem-dead', 'memory', 'dead', 'MemoryBindings', 8, 'boom', 'permanent', datetime('now'))",
1098            [],
1099        )
1100        .unwrap();
1101        let n = conn
1102            .execute(
1103                "UPDATE queue SET status='pending', attempt=0, next_retry_at=NULL, \
1104                 error=NULL, error_class=NULL \
1105                 WHERE status='dead' AND (operation = ?1 OR operation IS NULL)",
1106                rusqlite::params!["MemoryBindings"],
1107            )
1108            .unwrap();
1109        assert_eq!(n, 1);
1110        let (status, attempt, nra): (String, i64, Option<String>) = conn
1111            .query_row(
1112                "SELECT status, attempt, next_retry_at FROM queue WHERE item_key='mem-dead'",
1113                [],
1114                |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
1115            )
1116            .unwrap();
1117        assert_eq!(status, "pending");
1118        assert_eq!(attempt, 0);
1119        assert!(nra.is_none());
1120        let _ = std::fs::remove_file(&path);
1121    }
1122
1123    #[test]
1124    fn skipped_item_keys_excludes_only_skipped_for_operation() {
1125        // GAP-SG-69: the body-enrich scan must drop memories already vetoed
1126        // `status='skipped'` so `--until-empty` converges instead of re-scanning a
1127        // non-expandable short body forever (the detached worker reported a
1128        // stuck backlog for 30+ min).
1129        let (conn, path) = open_temp_queue();
1130        conn.execute(
1131            "INSERT INTO queue (item_key, item_type, status, operation) VALUES ('mem-vetoed', 'memory', 'skipped', 'BodyEnrich')",
1132            [],
1133        )
1134        .unwrap();
1135        conn.execute(
1136            "INSERT INTO queue (item_key, item_type, status, operation) VALUES ('mem-pending', 'memory', 'pending', 'BodyEnrich')",
1137            [],
1138        )
1139        .unwrap();
1140        conn.execute(
1141            "INSERT INTO queue (item_key, item_type, status, operation) VALUES ('mem-other-op', 'memory', 'skipped', 'MemoryBindings')",
1142            [],
1143        )
1144        .unwrap();
1145        let keys = skipped_item_keys(&conn, "BodyEnrich").unwrap();
1146        assert!(
1147            keys.contains("mem-vetoed"),
1148            "vetoed BodyEnrich item must be excluded from scan"
1149        );
1150        assert!(
1151            !keys.contains("mem-pending"),
1152            "pending item is still actionable"
1153        );
1154        assert!(
1155            !keys.contains("mem-other-op"),
1156            "skipped item from another operation must not leak"
1157        );
1158        assert_eq!(keys.len(), 1);
1159        let _ = std::fs::remove_file(&path);
1160    }
1161
1162    #[test]
1163    fn cascade_cleanup_delete_targets_memory_id_and_name() {
1164        let (conn, path) = open_temp_queue();
1165        conn.execute(
1166            "INSERT INTO queue (item_key, item_type, status, memory_id) VALUES ('by-id', 'memory', 'done', 42)",
1167            [],
1168        )
1169        .unwrap();
1170        conn.execute(
1171            "INSERT INTO queue (item_key, item_type, status) VALUES ('by-name', 'memory', 'pending')",
1172            [],
1173        )
1174        .unwrap();
1175        let removed = conn
1176            .execute(
1177                "DELETE FROM queue WHERE memory_id = ?1 OR item_key = ?2",
1178                rusqlite::params![42_i64, "by-name"],
1179            )
1180            .unwrap();
1181        assert_eq!(removed, 2);
1182        let remaining: i64 = conn
1183            .query_row("SELECT COUNT(*) FROM queue", [], |r| r.get(0))
1184            .unwrap();
1185        assert_eq!(remaining, 0);
1186        let _ = std::fs::remove_file(&path);
1187    }
1188
1189    #[test]
1190    fn item_type_for_maps_entity_and_memory() {
1191        assert_eq!(
1192            item_type_for(&EnrichOperation::EntityDescriptions),
1193            "entity"
1194        );
1195        assert_eq!(item_type_for(&EnrichOperation::MemoryBindings), "memory");
1196        assert_eq!(item_type_for(&EnrichOperation::AugmentBindings), "memory");
1197        assert_eq!(item_type_for(&EnrichOperation::BodyExtract), "memory");
1198    }
1199
1200    // v1.1.1 (P2): prefixed re-embed keys override the operation default so
1201    // prune_dead_orphans never reaps entity/chunk rows as orphaned memories.
1202    #[test]
1203    fn item_type_for_key_honours_reembed_prefixes() {
1204        assert_eq!(item_type_for_key("plain-memory-name", "memory"), "memory");
1205        assert_eq!(
1206            item_type_for_key("entity:tokio-runtime", "memory"),
1207            "entity"
1208        );
1209        assert_eq!(item_type_for_key("chunk:42", "memory"), "chunk");
1210        assert_eq!(item_type_for_key("some-entity", "entity"), "entity");
1211    }
1212
1213    #[test]
1214    fn prune_dead_orphans_removes_only_orphan_memory_rows() {
1215        let main = open_test_db();
1216        // One live memory whose dead row must be KEPT (it still exists).
1217        main.execute(
1218            "INSERT INTO memories (namespace, name, body) VALUES ('global', 'alive', 'b')",
1219            [],
1220        )
1221        .unwrap();
1222        let (queue, path) = open_temp_queue();
1223        // Orphan dead memory row (no matching memory) -> pruned.
1224        queue
1225            .execute(
1226                "INSERT INTO queue (item_key, item_type, status, operation, error_class) \
1227                 VALUES ('gone', 'memory', 'dead', 'MemoryBindings', 'permanent')",
1228                [],
1229            )
1230            .unwrap();
1231        // Live dead memory row (memory exists) -> kept.
1232        queue
1233            .execute(
1234                "INSERT INTO queue (item_key, item_type, status, operation, error_class) \
1235                 VALUES ('alive', 'memory', 'dead', 'MemoryBindings', 'permanent')",
1236                [],
1237            )
1238            .unwrap();
1239        // Entity dead row -> never touched (key is not a memory name).
1240        queue
1241            .execute(
1242                "INSERT INTO queue (item_key, item_type, status, operation) \
1243                 VALUES ('some-entity', 'entity', 'dead', 'EntityDescriptions')",
1244                [],
1245            )
1246            .unwrap();
1247
1248        let pruned = prune_dead_orphans(&queue, &main, "MemoryBindings", "global").unwrap();
1249        assert_eq!(pruned, 1, "only the orphan memory row is pruned");
1250
1251        let remaining: Vec<String> = {
1252            let mut stmt = queue
1253                .prepare("SELECT item_key FROM queue ORDER BY item_key")
1254                .unwrap();
1255            stmt.query_map([], |r| r.get::<_, String>(0))
1256                .unwrap()
1257                .collect::<Result<Vec<_>, _>>()
1258                .unwrap()
1259        };
1260        assert_eq!(remaining, vec!["alive", "some-entity"]);
1261        let _ = std::fs::remove_file(&path);
1262    }
1263
1264    #[test]
1265    fn prune_dead_entity_orphans_removes_only_entity_dead_rows() {
1266        let (queue, path) = open_temp_queue();
1267        // Entity dead row -> pruned (terminal artifact, no recovery path).
1268        queue
1269            .execute(
1270                "INSERT INTO queue (item_key, item_type, status, operation, error_class) \
1271                 VALUES ('entity:foo', 'entity', 'dead', 'ReEmbed', 'permanent')",
1272                [],
1273            )
1274            .unwrap();
1275        // Memory dead row -> untouched (wrong item_type).
1276        queue
1277            .execute(
1278                "INSERT INTO queue (item_key, item_type, status, operation, error_class) \
1279                 VALUES ('mem-dead', 'memory', 'dead', 'MemoryBindings', 'permanent')",
1280                [],
1281            )
1282            .unwrap();
1283        // Entity pending row -> untouched (not dead).
1284        queue
1285            .execute(
1286                "INSERT INTO queue (item_key, item_type, status, operation) \
1287                 VALUES ('entity:bar', 'entity', 'pending', 'ReEmbed')",
1288                [],
1289            )
1290            .unwrap();
1291
1292        let pruned = prune_dead_entity_orphans(&queue, "ReEmbed").unwrap();
1293        assert_eq!(pruned, 1, "only the entity dead row is pruned");
1294
1295        let remaining: Vec<String> = {
1296            let mut stmt = queue
1297                .prepare("SELECT item_key FROM queue ORDER BY item_key")
1298                .unwrap();
1299            stmt.query_map([], |r| r.get::<_, String>(0))
1300                .unwrap()
1301                .collect::<Result<Vec<_>, _>>()
1302                .unwrap()
1303        };
1304        assert_eq!(remaining, vec!["entity:bar", "mem-dead"]);
1305        let _ = std::fs::remove_file(&path);
1306    }
1307}