1mod extraction;
26mod postprocess;
27mod queue;
28mod scan;
29use extraction::{
30 call_body_enrich, call_body_extract, call_deep_research_synth, call_description_enrich,
31 call_domain_classify, call_entity_connect, call_entity_description, call_entity_type_validate,
32 call_graph_audit, call_memory_bindings, call_reembed, call_relation_reclassify,
33 call_weight_calibrate, find_codex_binary, take_last_openrouter_failure, EnrichItemResult,
34};
35use postprocess::{
36 persist_enriched_body, persist_entity_description, persist_memory_bindings,
37 reembed_memory_vector, take_enrich_backend,
38};
39pub use queue::{cleanup_queue_entry, DeadItem, DeadSummary, EnrichStatus, WaitingItem};
40use queue::{
41 dequeue_next_pending, enqueue_candidate, heartbeat, item_type_for, item_type_for_key,
42 open_queue_db, prune_dead_entity_orphans, prune_dead_orphans, record_item_failure,
43 record_item_failure_typed, reset_stale_processing_claims, skipped_item_keys, DequeueOutcome,
44};
45use scan::{count_operation_backlog, scan_operation, scan_unbound_memories};
46
47use crate::commands::ingest_claude::find_claude_binary;
48use crate::constants::MAX_MEMORY_BODY_LEN;
49use crate::entity_type::EntityType;
50use crate::errors::AppError;
51use crate::paths::AppPaths;
52use crate::storage::connection::{ensure_db_ready, open_rw};
53use crate::storage::entities::{self, NewEntity, NewRelationship};
54use crate::storage::memories;
55
56use rusqlite::Connection;
57use serde::{Deserialize, Serialize};
58use std::io::Write;
59use std::path::{Path, PathBuf};
60use std::time::Instant;
61
62const DEFAULT_RATE_LIMIT_WAIT: u64 = 60;
67const DEFAULT_BODY_ENRICH_MIN_CHARS: usize = 500;
68const DEFAULT_BODY_ENRICH_MAX_CHARS: usize = 2000;
69
70const BINDINGS_SCHEMA: &str = r#"{
75 "type": "object",
76 "properties": {
77 "entities": {
78 "type": "array",
79 "items": {
80 "type": "object",
81 "properties": {
82 "name": { "type": "string" },
83 "entity_type": {
84 "type": "string",
85 "enum": ["project","tool","person","file","concept","incident","decision","organization","location","date"]
86 }
87 },
88 "required": ["name", "entity_type"],
89 "additionalProperties": false
90 }
91 },
92 "relationships": {
93 "type": "array",
94 "items": {
95 "type": "object",
96 "properties": {
97 "source": { "type": "string" },
98 "target": { "type": "string" },
99 "relation": {
100 "type": "string",
101 "enum": ["applies-to","uses","depends-on","causes","fixes","contradicts","supports","follows","related","replaces","tracked-in"]
102 },
103 "strength": { "type": "number", "minimum": 0, "maximum": 1 }
104 },
105 "required": ["source","target","relation","strength"],
106 "additionalProperties": false
107 }
108 }
109 },
110 "required": ["entities","relationships"],
111 "additionalProperties": false
112}"#;
113
114const ENTITY_DESCRIPTION_SCHEMA: &str = r#"{
115 "type": "object",
116 "properties": {
117 "description": { "type": "string" }
118 },
119 "required": ["description"],
120 "additionalProperties": false
121}"#;
122
123const BODY_ENRICH_SCHEMA: &str = r#"{
124 "type": "object",
125 "properties": {
126 "enriched_body": { "type": "string" }
127 },
128 "required": ["enriched_body"],
129 "additionalProperties": false
130}"#;
131
132const WEIGHT_CALIBRATE_PROMPT: &str = "You are a knowledge graph quality auditor. Evaluate whether this relationship weight is correctly calibrated.\n\n\
134Scale:\n\
135- 0.9 = vital hard dependency (A cannot function without B)\n\
136- 0.7 = important design relationship (A strongly supports/enables B)\n\
137- 0.5 = useful contextual link (A and B share relevant context)\n\
138- 0.3 = weak reference (A mentions B without strong coupling)\n\n\
139Respond with the calibrated weight and brief reasoning.";
140
141const WEIGHT_CALIBRATE_SCHEMA: &str = r#"{
142 "type": "object",
143 "properties": {
144 "calibrated_weight": { "type": "number", "minimum": 0.0, "maximum": 1.0 },
145 "reasoning": { "type": "string" }
146 },
147 "required": ["calibrated_weight", "reasoning"],
148 "additionalProperties": false
149}"#;
150
151const RELATION_RECLASSIFY_PROMPT: &str = "You are a knowledge graph quality auditor. The relationship between these entities uses a generic type. Determine the REAL semantic relationship.\n\n\
153Valid canonical relations (pick exactly one):\n\
154- depends-on: A cannot function without B\n\
155- uses: A utilizes B but could substitute it\n\
156- supports: A reinforces or enables B\n\
157- causes: A triggers or produces B\n\
158- fixes: A resolves a problem in B\n\
159- contradicts: A conflicts with or invalidates B\n\
160- applies-to: A is relevant to or scoped within B\n\
161- follows: A comes after B in sequence\n\
162- replaces: A substitutes B\n\
163- tracked-in: A is monitored in B\n\
164- related: A and B share context (use sparingly)\n\n\
165Respond with the correct relation, strength, and reasoning.";
166
167const RELATION_RECLASSIFY_SCHEMA: &str = r#"{
168 "type": "object",
169 "properties": {
170 "relation": { "type": "string" },
171 "strength": { "type": "number", "minimum": 0.0, "maximum": 1.0 },
172 "reasoning": { "type": "string" }
173 },
174 "required": ["relation", "strength", "reasoning"],
175 "additionalProperties": false
176}"#;
177
178const ENTITY_CONNECT_PROMPT: &str = "You are a knowledge graph quality auditor. Two entities exist in the same graph but have no relationship between them. Determine if a meaningful relationship exists.\n\n\
180Valid canonical relations: depends-on, uses, supports, causes, fixes, contradicts, applies-to, follows, replaces, tracked-in, related.\n\n\
181If NO meaningful relationship exists, set relation to \"none\".\n\
182Respond with the relation (or \"none\"), strength, and reasoning.";
183
184const ENTITY_CONNECT_SCHEMA: &str = r#"{
185 "type": "object",
186 "properties": {
187 "relation": { "type": "string" },
188 "strength": { "type": "number", "minimum": 0.0, "maximum": 1.0 },
189 "reasoning": { "type": "string" }
190 },
191 "required": ["relation", "strength", "reasoning"],
192 "additionalProperties": false
193}"#;
194
195const ENTITY_TYPE_VALIDATE_PROMPT: &str = "You are a knowledge graph quality auditor. Verify whether this entity's type is correct.\n\n\
197Valid entity types: project, tool, person, file, concept, incident, decision, organization, location, date.\n\n\
198If the current type is correct, keep it. If wrong, suggest the correct type.\n\
199Respond with the validated type and reasoning.";
200
201const ENTITY_TYPE_VALIDATE_SCHEMA: &str = r#"{
202 "type": "object",
203 "properties": {
204 "validated_type": { "type": "string" },
205 "was_correct": { "type": "boolean" },
206 "reasoning": { "type": "string" }
207 },
208 "required": ["validated_type", "was_correct", "reasoning"],
209 "additionalProperties": false
210}"#;
211
212const DESCRIPTION_ENRICH_PROMPT: &str = "You are a knowledge graph quality auditor. This memory has a generic or auto-generated description. Write a concise, semantic description (10-20 words) that captures WHAT this memory is about and WHY it matters.\n\n\
214BAD: 'ingested from docs/auth.md'\n\
215GOOD: 'JWT token rotation strategy with 15-min expiry and refresh flow'\n\n\
216Respond with the improved description and reasoning.";
217
218const DESCRIPTION_ENRICH_SCHEMA: &str = r#"{
219 "type": "object",
220 "properties": {
221 "description": { "type": "string" },
222 "reasoning": { "type": "string" }
223 },
224 "required": ["description", "reasoning"],
225 "additionalProperties": false
226}"#;
227
228const DOMAIN_CLASSIFY_PROMPT: &str = "You are a knowledge graph quality auditor. Classify this memory into its primary domain category.\n\n\
230Respond with the domain name (kebab-case, 2-4 words) and reasoning.";
231
232const DOMAIN_CLASSIFY_SCHEMA: &str = r#"{
233 "type": "object",
234 "properties": {
235 "domain": { "type": "string" },
236 "confidence": { "type": "number", "minimum": 0.0, "maximum": 1.0 },
237 "reasoning": { "type": "string" }
238 },
239 "required": ["domain", "confidence", "reasoning"],
240 "additionalProperties": false
241}"#;
242
243const GRAPH_AUDIT_PROMPT: &str = "You are a knowledge graph quality auditor. Analyze this memory and its entity bindings for quality issues.\n\n\
245Check for: missing entities, wrong entity types, redundant relationships, orphaned entities, generic descriptions, low-signal relationships.\n\n\
246Respond with a list of issues found (or empty if none) and an overall quality score.";
247
248const GRAPH_AUDIT_SCHEMA: &str = r#"{
249 "type": "object",
250 "properties": {
251 "quality_score": { "type": "number", "minimum": 0.0, "maximum": 1.0 },
252 "issues": { "type": "array", "items": { "type": "object", "properties": { "kind": { "type": "string" }, "detail": { "type": "string" } }, "required": ["kind", "detail"] } },
253 "reasoning": { "type": "string" }
254 },
255 "required": ["quality_score", "issues", "reasoning"],
256 "additionalProperties": false
257}"#;
258
259const DEEP_RESEARCH_SYNTH_PROMPT: &str = "You are a knowledge graph synthesizer. Given this memory body, extract key findings and synthesize them into structured entities and relationships.\n\n\
261Entity names: lowercase kebab-case, domain-specific.\n\
262Relations: depends-on, uses, supports, causes, fixes, contradicts, applies-to, follows, related, replaces, tracked-in.\n\n\
263Respond with extracted entities, relationships, and a synthesis summary.";
264
265const DEEP_RESEARCH_SYNTH_SCHEMA: &str = r#"{
266 "type": "object",
267 "properties": {
268 "entities": { "type": "array", "items": { "type": "object", "properties": { "name": { "type": "string" }, "entity_type": { "type": "string" } }, "required": ["name", "entity_type"] } },
269 "relationships": { "type": "array", "items": { "type": "object", "properties": { "source": { "type": "string" }, "target": { "type": "string" }, "relation": { "type": "string" }, "strength": { "type": "number" } }, "required": ["source", "target", "relation", "strength"] } },
270 "summary": { "type": "string" }
271 },
272 "required": ["entities", "relationships", "summary"],
273 "additionalProperties": false
274}"#;
275
276const BODY_EXTRACT_PROMPT: &str = "You are a structured data extractor. Given this memory body (which may be unstructured text, raw notes, or a transcript), extract and restructure the content into a clean, well-organized markdown body.\n\n\
278Preserve all factual content. Remove noise, fix formatting, add section headers where appropriate.\n\
279Respond with the restructured body and a brief summary of changes.";
280
281const BODY_EXTRACT_SCHEMA: &str = r#"{
282 "type": "object",
283 "properties": {
284 "restructured_body": { "type": "string" },
285 "changes_summary": { "type": "string" }
286 },
287 "required": ["restructured_body", "changes_summary"],
288 "additionalProperties": false
289}"#;
290
291const BINDINGS_PROMPT: &str = "You are a knowledge graph entity extractor. Given a memory body, extract:\n\
2961. Domain-specific entities (concepts, tools, people, decisions, projects, files)\n\
2972. Typed relationships between entities with strength scores\n\n\
298Rules:\n\
299- Entity names: lowercase kebab-case, 2+ chars, domain-specific only\n\
300- NEVER extract generic terms, stop words, numbers, UUIDs, or single characters\n\
301- Relationship types MUST be one of: applies-to, uses, depends-on, causes, fixes, contradicts, supports, follows, related, replaces, tracked-in\n\
302- NEVER use 'mentions' as relationship type\n\
303- Strength: 0.9 for hard dependencies, 0.7 for design relationships, 0.5 for contextual links, 0.3 for weak references\n\
304- Prefer fewer high-quality entities over many low-quality ones";
305
306const ENTITY_DESCRIPTION_PROMPT_PREFIX: &str = "You are a knowledge graph annotator. Given an entity name and type, write a concise one-sentence description (10-20 words) that explains what this entity IS and WHY it matters in the context of software/system design.\n\nEntity name: ";
307
308const BODY_ENRICH_PROMPT_PREFIX: &str = "You are a knowledge assistant. Given a short or sparse memory body, expand it into a richer, more complete and useful description. Preserve all existing facts. Add context, implications, and relationships that would be valuable for knowledge retrieval.\n\nConstraints:\n- Output only the enriched body text (no metadata, no headers)\n- Preserve the original meaning exactly\n- Target length is provided in the system context\n\nMemory body to enrich:\n\n";
309
310#[derive(Debug, Clone, PartialEq, Eq, clap::ValueEnum, Serialize, Deserialize)]
316#[serde(rename_all = "kebab-case")]
317pub enum EnrichOperation {
318 MemoryBindings,
323 AugmentBindings,
328 EntityDescriptions,
330 BodyEnrich,
332 ReEmbed,
334 WeightCalibrate,
336 RelationReclassify,
338 EntityConnect,
340 EntityTypeValidate,
342 DescriptionEnrich,
344 CrossDomainBridges,
348 DomainClassify,
350 GraphAudit,
352 DeepResearchSynth,
354 BodyExtract,
356}
357
358#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum, Serialize, Deserialize)]
365#[serde(rename_all = "kebab-case")]
366pub enum ReEmbedTarget {
367 Memories,
369 Entities,
371 Chunks,
373 All,
375}
376
377#[derive(Debug, Clone, PartialEq, Eq, clap::ValueEnum)]
379pub enum EnrichMode {
380 ClaudeCode,
382 Codex,
384 #[value(name = "opencode")]
386 Opencode,
387 #[value(name = "openrouter")]
389 OpenRouter,
390}
391
392impl std::fmt::Display for EnrichMode {
393 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
394 match self {
395 EnrichMode::ClaudeCode => write!(f, "claude-code"),
396 EnrichMode::Codex => write!(f, "codex"),
397 EnrichMode::Opencode => write!(f, "opencode"),
398 EnrichMode::OpenRouter => write!(f, "openrouter"),
399 }
400 }
401}
402
403#[derive(clap::Args)]
405#[command(
406 about = "Enrich graph memories and entities using an LLM provider",
407 after_long_help = "EXAMPLES:\n \
408 # Add missing entity bindings to all unbound memories\n \
409 sqlite-graphrag enrich --operation memory-bindings --mode codex --codex-model gpt-5.4-mini\n\n \
410 # Fill entity descriptions (dry-run preview, no tokens spent)\n \
411 sqlite-graphrag enrich --operation entity-descriptions --dry-run --json\n\n \
412 # Expand short memory bodies (GAP-18)\n \
413 sqlite-graphrag enrich --operation body-enrich --min-output-chars 600\n\n \
414 # Rebuild only missing memory embeddings without rewriting bodies\n \
415 sqlite-graphrag enrich --operation re-embed --limit 100\n\n \
416 # Resume an interrupted body-enrich run\n \
417 sqlite-graphrag enrich --operation body-enrich --resume --json\n\n \
418 # Retry only failed items from a previous run\n \
419 sqlite-graphrag enrich --operation memory-bindings --retry-failed --json\n\n \
420 # Converge the whole backlog (internal scan+drain loop, no bash wrapper)\n \
421 sqlite-graphrag enrich --operation memory-bindings --mode openrouter \\\n \
422 --openrouter-model deepseek/deepseek-v4-flash:nitro --until-empty --max-runtime 600\n\n \
423 # Inspect / resurrect dead-letter items\n \
424 sqlite-graphrag enrich --operation memory-bindings --list-dead\n \
425 sqlite-graphrag enrich --operation memory-bindings --requeue-dead\n\n \
426 # Read-only status (no LLM, no singleton)\n \
427 sqlite-graphrag enrich --operation memory-bindings --status\n\n\
428 OPERATIONS NOTE:\n \
429 memory-bindings LINKS each memory to the EXISTING entities extracted from its\n \
430 body — it does not invent a new graph, it connects what is missing. It scans\n \
431 only UNBOUND memories. To re-run extraction over ALREADY-bound memories and\n \
432 MERGE newly-found entities/relationships additively (without removing links),\n \
433 use --operation augment-bindings with --names/--names-file.\n\n\
434 DEAD-LETTER SIDECAR (.enrich-queue.sqlite):\n \
435 A SQLite sidecar tracks each work item across runs. Schema (table `queue`):\n \
436 item_key (UNIQUE name/id), item_type (memory|entity), operation, memory_id,\n \
437 status (pending|processing|done|skipped|dead), attempt, error, error_class,\n \
438 next_retry_at (backoff cooldown). --until-empty loops scan→drain internally\n \
439 until eligible items are exhausted; transient failures (incl. malformed/non-\n \
440 JSON LLM output, GAP-SG-09) reschedule with backoff until --max-attempts, then\n \
441 land in status='dead'. Use --status to see the queue, --list-dead to inspect\n \
442 the sink, --requeue-dead to retry it, and --ignore-backoff to skip cooldowns.\n \
443 --names/--names-file also remedy a cooldown by targeting a specific subset.\n\n\
444 EXIT CODES:\n \
445 0 success\n \
446 1 validation error (bad args, binary not found)\n \
447 14 I/O error"
448)]
449pub struct EnrichArgs {
450 #[arg(
455 long,
456 short = 'o',
457 value_enum,
458 value_name = "OPERATION",
459 required_unless_present_any = ["status", "list_dead", "requeue_dead", "prune_dead_orphans", "prune_dead_entity_orphans"]
460 )]
461 pub operation: Option<EnrichOperation>,
462
463 #[arg(
467 long,
468 value_enum,
469 required_unless_present_any = ["status", "list_dead", "requeue_dead", "prune_dead_orphans", "prune_dead_entity_orphans"]
470 )]
471 pub mode: Option<EnrichMode>,
472
473 #[arg(long, value_name = "N")]
475 pub limit: Option<usize>,
476
477 #[arg(long, value_enum, value_name = "TARGET", default_value_t = ReEmbedTarget::Memories)]
485 pub target: ReEmbedTarget,
486
487 #[arg(long)]
489 pub dry_run: bool,
490
491 #[arg(long, env = "SQLITE_GRAPHRAG_NAMESPACE")]
493 pub namespace: Option<String>,
494
495 #[arg(long, value_name = "PATH")]
498 pub claude_binary: Option<PathBuf>,
499
500 #[arg(long, value_name = "MODEL")]
502 pub claude_model: Option<String>,
503
504 #[arg(long, value_name = "SECONDS", default_value_t = 300)]
506 pub claude_timeout: u64,
507
508 #[arg(long, value_name = "PATH")]
511 pub codex_binary: Option<PathBuf>,
512
513 #[arg(long, value_name = "MODEL")]
515 pub codex_model: Option<String>,
516
517 #[arg(long, value_name = "SECONDS", default_value_t = 300)]
519 pub codex_timeout: u64,
520
521 #[arg(long, value_name = "PATH", env = "SQLITE_GRAPHRAG_OPENCODE_BINARY")]
524 pub opencode_binary: Option<PathBuf>,
525
526 #[arg(long, value_name = "MODEL", env = "SQLITE_GRAPHRAG_OPENCODE_MODEL")]
528 pub opencode_model: Option<String>,
529
530 #[arg(
532 long,
533 value_name = "SECONDS",
534 env = "SQLITE_GRAPHRAG_OPENCODE_TIMEOUT",
535 default_value_t = 300
536 )]
537 pub opencode_timeout: u64,
538
539 #[arg(long, value_name = "MODEL")]
542 pub openrouter_model: Option<String>,
543
544 #[arg(
546 long,
547 value_name = "KEY",
548 env = "OPENROUTER_API_KEY",
549 hide_env_values = true
550 )]
551 pub openrouter_api_key: Option<String>,
552
553 #[arg(long, value_name = "SECONDS", default_value_t = 600)]
560 pub openrouter_timeout: u64,
561
562 #[arg(long, value_name = "URL")]
564 pub openrouter_base_url: Option<String>,
565
566 #[arg(long, value_name = "USD")]
569 pub max_cost_usd: Option<f64>,
570
571 #[arg(long)]
574 pub resume: bool,
575
576 #[arg(long)]
578 pub retry_failed: bool,
579
580 #[arg(long)]
584 pub reset_stale_claims: bool,
585
586 #[arg(long, value_name = "SECONDS", default_value_t = 1800)]
589 pub stale_claim_secs: u64,
590
591 #[arg(long)]
595 pub until_empty: bool,
596
597 #[arg(long, value_name = "SECONDS")]
600 pub max_runtime: Option<u64>,
601
602 #[arg(long, value_name = "N", default_value_t = 8, value_parser = clap::value_parser!(u32).range(1..=20))]
614 pub max_attempts: u32,
615
616 #[arg(long)]
630 pub status: bool,
631
632 #[arg(long)]
637 pub list_dead: bool,
638
639 #[arg(long)]
646 pub requeue_dead: bool,
647
648 #[arg(long)]
656 pub prune_dead_orphans: bool,
657
658 #[arg(long, conflicts_with = "prune_dead_orphans")]
666 pub prune_dead_entity_orphans: bool,
667
668 #[arg(long)]
674 pub ignore_backoff: bool,
675
676 #[arg(long)]
683 pub body_extract_graph_only: bool,
684
685 #[arg(long, value_name = "N", value_parser = clap::value_parser!(u32).range(1..=16))]
688 pub rest_concurrency: Option<u32>,
689
690 #[arg(long, value_name = "CHARS", default_value_t = DEFAULT_BODY_ENRICH_MIN_CHARS)]
693 pub min_output_chars: usize,
694
695 #[arg(long, value_name = "CHARS", default_value_t = DEFAULT_BODY_ENRICH_MAX_CHARS)]
697 pub max_output_chars: usize,
698
699 #[arg(long, default_value_t = true)]
701 pub preserve_check: bool,
702
703 #[arg(long, value_name = "PATH")]
705 pub prompt_template: Option<PathBuf>,
706
707 #[arg(long, default_value_t = 1, value_name = "N", value_parser = clap::value_parser!(u32).range(1..=32))]
711 pub llm_parallelism: u32,
712
713 #[arg(long)]
716 pub json: bool,
717
718 #[arg(long, env = "SQLITE_GRAPHRAG_DB_PATH")]
720 pub db: Option<String>,
721
722 #[arg(long, value_name = "SECONDS")]
725 pub wait_job_singleton: Option<u64>,
726
727 #[arg(long, default_value_t = false)]
731 pub force_job_singleton: bool,
732
733 #[arg(long, value_name = "NAMES", value_delimiter = ',')]
743 pub names: Vec<String>,
744
745 #[arg(long, value_name = "PATH")]
749 pub names_file: Option<PathBuf>,
750
751 #[arg(long, default_value_t = false)]
755 pub preflight_check: bool,
756
757 #[arg(long, value_enum)]
761 pub fallback_mode: Option<EnrichMode>,
762
763 #[arg(long, value_name = "SECONDS", default_value_t = 300)]
766 pub rate_limit_buffer: u64,
767
768 #[arg(long, default_value_t = true)]
772 pub max_load_check: bool,
773
774 #[arg(long, value_name = "N", default_value_t = 5)]
777 pub circuit_breaker_threshold: u32,
778
779 #[arg(long, value_name = "FLOAT", default_value_t = 0.7)]
786 pub preserve_threshold: f64,
787
788 #[arg(long, default_value_t = true)]
793 pub codex_model_validate: bool,
794
795 #[arg(long, value_name = "MODEL")]
800 pub codex_model_fallback: Option<String>,
801}
802
803impl EnrichArgs {
804 fn operation(&self) -> EnrichOperation {
812 self.operation
813 .clone()
814 .unwrap_or(EnrichOperation::MemoryBindings)
815 }
816
817 fn mode(&self) -> EnrichMode {
822 self.mode.clone().unwrap_or(EnrichMode::OpenRouter)
823 }
824}
825
826#[derive(Debug, Serialize)]
835struct PhaseEvent<'a> {
836 phase: &'a str,
837 #[serde(skip_serializing_if = "Option::is_none")]
838 binary_path: Option<&'a str>,
839 #[serde(skip_serializing_if = "Option::is_none")]
840 version: Option<&'a str>,
841 #[serde(skip_serializing_if = "Option::is_none")]
842 items_total: Option<usize>,
843 #[serde(skip_serializing_if = "Option::is_none")]
844 items_pending: Option<usize>,
845 #[serde(skip_serializing_if = "Option::is_none")]
847 llm_parallelism: Option<u32>,
848}
849
850#[derive(Debug, Serialize)]
853struct ScanStartEvent<'a> {
854 phase: &'static str,
855 operation: &'a str,
857 entities_in_namespace: i64,
858 backlog_degree0_proxy: Option<i64>,
860 pair_algorithm: Option<&'static str>,
861 limit: Option<usize>,
862 scan_deadline_secs: Option<u64>,
863}
864
865fn enrich_operation_cli_name(op: &EnrichOperation) -> &'static str {
867 match op {
868 EnrichOperation::MemoryBindings => "memory-bindings",
869 EnrichOperation::AugmentBindings => "augment-bindings",
870 EnrichOperation::EntityDescriptions => "entity-descriptions",
871 EnrichOperation::BodyEnrich => "body-enrich",
872 EnrichOperation::ReEmbed => "re-embed",
873 EnrichOperation::WeightCalibrate => "weight-calibrate",
874 EnrichOperation::RelationReclassify => "relation-reclassify",
875 EnrichOperation::EntityConnect => "entity-connect",
876 EnrichOperation::EntityTypeValidate => "entity-type-validate",
877 EnrichOperation::DescriptionEnrich => "description-enrich",
878 EnrichOperation::CrossDomainBridges => "cross-domain-bridges",
879 EnrichOperation::DomainClassify => "domain-classify",
880 EnrichOperation::GraphAudit => "graph-audit",
881 EnrichOperation::DeepResearchSynth => "deep-research-synth",
882 EnrichOperation::BodyExtract => "body-extract",
883 }
884}
885
886#[derive(Debug, Serialize)]
891struct ConcurrencyEvent {
892 phase: &'static str,
893 scan_parallelism: u32,
894 drain_parallelism: u32,
895}
896
897#[derive(Debug, Serialize)]
898struct ItemEvent<'a> {
899 item: &'a str,
901 status: &'a str,
902 #[serde(skip_serializing_if = "Option::is_none")]
903 memory_id: Option<i64>,
904 #[serde(skip_serializing_if = "Option::is_none")]
905 entity_id: Option<i64>,
906 #[serde(skip_serializing_if = "Option::is_none")]
907 entities: Option<usize>,
908 #[serde(skip_serializing_if = "Option::is_none")]
909 rels: Option<usize>,
910 #[serde(skip_serializing_if = "Option::is_none")]
911 chars_before: Option<usize>,
912 #[serde(skip_serializing_if = "Option::is_none")]
913 chars_after: Option<usize>,
914 #[serde(skip_serializing_if = "Option::is_none")]
915 cost_usd: Option<f64>,
916 #[serde(skip_serializing_if = "Option::is_none")]
917 elapsed_ms: Option<u64>,
918 #[serde(skip_serializing_if = "Option::is_none")]
919 error: Option<String>,
920 index: usize,
921 total: usize,
922}
923
924#[derive(Debug, Serialize)]
925struct EnrichSummary {
926 summary: bool,
927 operation: String,
928 items_total: usize,
929 completed: usize,
930 failed: usize,
931 skipped: usize,
932 cost_usd: f64,
933 elapsed_ms: u64,
934 #[serde(skip_serializing_if = "Option::is_none")]
939 backend_invoked: Option<&'static str>,
940 waiting: i64,
944 dead: i64,
947}
948
949use crate::output::emit_json_line as emit_json;
950
951enum PreflightOutcome {
965 Healthy,
967 RateLimited {
971 reason: String,
972 suggestion: &'static str,
973 },
974 Error(AppError),
976}
977
978fn run_preflight_probe(args: &EnrichArgs) -> PreflightOutcome {
986 let timeout = std::time::Duration::from_secs(args.rate_limit_buffer.max(60));
987
988 match args.mode() {
989 EnrichMode::ClaudeCode => {
990 let bin = match find_claude_binary(args.claude_binary.as_deref()) {
991 Ok(b) => b,
992 Err(e) => return PreflightOutcome::Error(e),
993 };
994 let mcp_config_path = match crate::spawn::preflight::write_empty_mcp_config_tempfile() {
999 Ok(p) => p,
1000 Err(e) => {
1001 return PreflightOutcome::Error(AppError::Io(e));
1002 }
1003 };
1004 let mut cmd = std::process::Command::new(&bin);
1005 crate::spawn::env_whitelist::apply_env_whitelist(
1006 &mut cmd,
1007 crate::spawn::env_whitelist::is_strict_env_clear(),
1008 );
1009 if let Err(e) = crate::spawn::apply_cwd_isolation(&mut cmd) {
1010 return PreflightOutcome::Error(e);
1011 }
1012 cmd.arg("-p")
1013 .arg("ping")
1014 .arg("--max-turns")
1015 .arg("1")
1016 .arg("--strict-mcp-config")
1017 .arg("--mcp-config")
1018 .arg(mcp_config_path.as_os_str())
1019 .arg("--dangerously-skip-permissions")
1020 .arg("--settings")
1021 .arg("{\"hooks\":{}}")
1022 .arg("--output-format")
1023 .arg("json")
1024 .stdin(std::process::Stdio::null())
1025 .stdout(std::process::Stdio::piped())
1026 .stderr(std::process::Stdio::piped());
1027
1028 let child = match super::claude_runner::spawn_with_memory_limit(&mut cmd) {
1029 Ok(c) => c,
1030 Err(e) => {
1031 return PreflightOutcome::Error(AppError::Io(e));
1032 }
1033 };
1034 let output = match wait_with_timeout(child, timeout) {
1035 Ok(out) => out,
1036 Err(e) => return PreflightOutcome::Error(e),
1037 };
1038 if !output.status.success() {
1039 let stderr = String::from_utf8_lossy(&output.stderr);
1040 if stderr.contains("hit your session limit")
1041 || stderr.contains("rate_limit")
1042 || stderr.contains("429")
1043 {
1044 return PreflightOutcome::RateLimited {
1045 reason: stderr.trim().to_string(),
1046 suggestion:
1047 "wait for the OAuth window to reset or use --fallback-mode codex",
1048 };
1049 }
1050 return PreflightOutcome::Error(AppError::Validation(format!(
1051 "preflight probe failed: {stderr}",
1052 stderr = stderr.trim()
1053 )));
1054 }
1055 PreflightOutcome::Healthy
1056 }
1057 EnrichMode::Codex => {
1058 let bin = match find_codex_binary(args.codex_binary.as_deref()) {
1059 Ok(b) => b,
1060 Err(e) => return PreflightOutcome::Error(e),
1061 };
1062 super::codex_spawn::validate_codex_model(args.codex_model.as_deref())
1063 .map_err(PreflightOutcome::Error)
1064 .ok();
1065 let schema = "{}";
1066 let schema_path = match super::codex_spawn::trusted_schema_path() {
1067 Ok(p) => p,
1068 Err(e) => return PreflightOutcome::Error(e),
1069 };
1070 let spawn_args = super::codex_spawn::CodexSpawnArgs {
1071 binary: &bin,
1072 prompt: "ping",
1073 json_schema: schema,
1074 input_text: "",
1075 model: args.codex_model.as_deref(),
1076 timeout_secs: args.rate_limit_buffer.max(60),
1077 schema_path: schema_path.clone(),
1078 };
1079 let mut cmd = match super::codex_spawn::build_codex_command(&spawn_args) {
1080 Ok(c) => c,
1081 Err(e) => return PreflightOutcome::Error(e),
1082 };
1083 let child = match super::claude_runner::spawn_with_memory_limit(&mut cmd) {
1084 Ok(c) => c,
1085 Err(e) => return PreflightOutcome::Error(AppError::Io(e)),
1086 };
1087 let output = match wait_with_timeout(child, timeout) {
1088 Ok(out) => out,
1089 Err(e) => return PreflightOutcome::Error(e),
1090 };
1091 let _ = std::fs::remove_file(&schema_path);
1092 if !output.status.success() {
1093 let stderr = String::from_utf8_lossy(&output.stderr);
1094 if stderr.contains("rate_limit")
1095 || stderr.contains("429")
1096 || stderr.contains("Too Many Requests")
1097 {
1098 return PreflightOutcome::RateLimited {
1099 reason: stderr.trim().to_string(),
1100 suggestion: "wait for the rate-limit window to reset",
1101 };
1102 }
1103 return PreflightOutcome::Error(AppError::Validation(format!(
1104 "preflight probe failed: {stderr}",
1105 stderr = stderr.trim()
1106 )));
1107 }
1108 PreflightOutcome::Healthy
1109 }
1110 EnrichMode::Opencode => {
1111 let bin = match super::opencode_runner::find_opencode_binary_with_override(
1112 args.opencode_binary.as_deref(),
1113 ) {
1114 Ok(b) => b,
1115 Err(e) => return PreflightOutcome::Error(e),
1116 };
1117 let model =
1118 super::opencode_runner::resolve_opencode_model(args.opencode_model.as_deref());
1119 let mut cmd =
1120 match super::opencode_runner::build_opencode_command_sync(&bin, &model, "ping", "")
1121 {
1122 Ok(c) => c,
1123 Err(e) => return PreflightOutcome::Error(e),
1124 };
1125 let child = match super::opencode_runner::spawn_opencode(&mut cmd) {
1126 Ok(c) => c,
1127 Err(e) => return PreflightOutcome::Error(AppError::Io(e)),
1128 };
1129 let output = match wait_with_timeout(child, timeout) {
1130 Ok(out) => out,
1131 Err(e) => return PreflightOutcome::Error(e),
1132 };
1133 if !output.status.success() {
1134 let stderr = String::from_utf8_lossy(&output.stderr);
1135 if stderr.contains("rate_limit")
1136 || stderr.contains("429")
1137 || stderr.contains("Too Many Requests")
1138 {
1139 return PreflightOutcome::RateLimited {
1140 reason: stderr.trim().to_string(),
1141 suggestion: "wait for the rate-limit window to reset",
1142 };
1143 }
1144 return PreflightOutcome::Error(AppError::Validation(format!(
1145 "preflight probe failed: {stderr}",
1146 stderr = stderr.trim()
1147 )));
1148 }
1149 PreflightOutcome::Healthy
1150 }
1151 EnrichMode::OpenRouter => {
1152 match crate::config::resolve_api_key("openrouter", args.openrouter_api_key.as_deref()) {
1156 Some(_) => PreflightOutcome::Healthy,
1157 None => PreflightOutcome::Error(AppError::Validation(
1158 "OPENROUTER_API_KEY not found for --mode openrouter preflight".into(),
1159 )),
1160 }
1161 }
1162 }
1163}
1164
1165fn wait_with_timeout(
1167 mut child: std::process::Child,
1168 timeout: std::time::Duration,
1169) -> Result<std::process::Output, AppError> {
1170 use wait_timeout::ChildExt;
1171 let start = std::time::Instant::now();
1172 let Some(exit) = child.wait_timeout(timeout).map_err(AppError::Io)? else {
1173 let _ = child.kill();
1174 let _ = child.wait();
1175 return Err(AppError::Validation(format!(
1176 "preflight probe timed out after {}s",
1177 start.elapsed().as_secs()
1178 )));
1179 };
1180 let mut stdout = Vec::new();
1181 if let Some(mut out) = child.stdout.take() {
1182 std::io::Read::read_to_end(&mut out, &mut stdout).map_err(AppError::Io)?;
1183 }
1184 let mut stderr = Vec::new();
1185 if let Some(mut err) = child.stderr.take() {
1186 std::io::Read::read_to_end(&mut err, &mut stderr).map_err(AppError::Io)?;
1187 }
1188 Ok(std::process::Output {
1189 status: exit,
1190 stdout,
1191 stderr,
1192 })
1193}
1194
1195fn is_at_default<T: PartialEq>(value: T, default: T) -> bool {
1211 value == default
1212}
1213
1214fn validate_mode_conditional_flags_enrich(args: &EnrichArgs) -> Result<(), AppError> {
1229 const DEFAULT_TIMEOUT: u64 = 300;
1230
1231 let mut conflicts: Vec<String> = Vec::new();
1232
1233 match args.mode() {
1234 EnrichMode::ClaudeCode => {
1235 if args.codex_binary.is_some() {
1236 conflicts.push("--codex-binary is ignored when --mode=claude-code".to_string());
1237 }
1238 if args.codex_model.is_some() {
1239 conflicts.push("--codex-model is ignored when --mode=claude-code".to_string());
1240 }
1241 if !is_at_default(args.codex_timeout, DEFAULT_TIMEOUT) {
1242 conflicts.push(format!(
1243 "--codex-timeout={} is ignored when --mode=claude-code (remove the flag to use the default 300s)",
1244 args.codex_timeout
1245 ));
1246 }
1247 }
1248 EnrichMode::Codex => {
1249 if args.claude_binary.is_some() {
1250 conflicts.push("--claude-binary is ignored when --mode=codex".to_string());
1251 }
1252 if args.claude_model.is_some() {
1253 conflicts.push("--claude-model is ignored when --mode=codex".to_string());
1254 }
1255 if !is_at_default(args.claude_timeout, DEFAULT_TIMEOUT) {
1256 conflicts.push(format!(
1257 "--claude-timeout={} is ignored when --mode=codex (remove the flag to use the default 300s)",
1258 args.claude_timeout
1259 ));
1260 }
1261 if args.max_cost_usd.is_some() {
1262 conflicts.push(
1263 "--max-cost-usd is ignored when --mode=codex (OAuth-first; cost is metered by your subscription, not the call)"
1264 .to_string(),
1265 );
1266 }
1267 }
1268 EnrichMode::Opencode => {
1269 if args.claude_binary.is_some() {
1270 conflicts.push("--claude-binary is ignored when --mode=opencode".to_string());
1271 }
1272 if args.claude_model.is_some() {
1273 conflicts.push("--claude-model is ignored when --mode=opencode".to_string());
1274 }
1275 if !is_at_default(args.claude_timeout, DEFAULT_TIMEOUT) {
1276 conflicts.push(format!(
1277 "--claude-timeout={} is ignored when --mode=opencode (remove the flag to use the default 300s)",
1278 args.claude_timeout
1279 ));
1280 }
1281 if args.max_cost_usd.is_some() {
1282 conflicts.push(
1283 "--max-cost-usd is ignored when --mode=opencode (OAuth-first; cost is metered by your subscription, not the call)"
1284 .to_string(),
1285 );
1286 }
1287 }
1288 EnrichMode::OpenRouter => {
1289 if args.claude_binary.is_some() {
1290 conflicts.push("--claude-binary is ignored when --mode=openrouter".to_string());
1291 }
1292 if args.claude_model.is_some() {
1293 conflicts.push("--claude-model is ignored when --mode=openrouter".to_string());
1294 }
1295 if args.codex_binary.is_some() {
1296 conflicts.push("--codex-binary is ignored when --mode=openrouter".to_string());
1297 }
1298 if args.codex_model.is_some() {
1299 conflicts.push("--codex-model is ignored when --mode=openrouter".to_string());
1300 }
1301 if args.opencode_binary.is_some() {
1302 conflicts.push("--opencode-binary is ignored when --mode=openrouter".to_string());
1303 }
1304 if args.opencode_model.is_some() {
1305 conflicts.push("--opencode-model is ignored when --mode=openrouter".to_string());
1306 }
1307 if !is_at_default(args.claude_timeout, DEFAULT_TIMEOUT) {
1308 conflicts.push(format!(
1309 "--claude-timeout={} is ignored when --mode=openrouter (remove the flag to use the default 300s)",
1310 args.claude_timeout
1311 ));
1312 }
1313 if !is_at_default(args.codex_timeout, DEFAULT_TIMEOUT) {
1314 conflicts.push(format!(
1315 "--codex-timeout={} is ignored when --mode=openrouter (remove the flag to use the default 300s)",
1316 args.codex_timeout
1317 ));
1318 }
1319 if !is_at_default(args.opencode_timeout, DEFAULT_TIMEOUT) {
1320 conflicts.push(format!(
1321 "--opencode-timeout={} is ignored when --mode=openrouter (remove the flag to use the default 300s)",
1322 args.opencode_timeout
1323 ));
1324 }
1325 }
1326 }
1327
1328 if !conflicts.is_empty() {
1329 return Err(AppError::Validation(format!(
1330 "G20: mode-conditional flag conflicts detected for --mode={}:\n - {}",
1331 args.mode(),
1332 conflicts.join("\n - ")
1333 )));
1334 }
1335
1336 Ok(())
1337}
1338
1339fn scan_operation_with_deadline(
1351 conn: &Connection,
1352 namespace: &str,
1353 args: &EnrichArgs,
1354 deadline: Option<Instant>,
1355) -> Result<Vec<String>, AppError> {
1356 let Some(deadline) = deadline else {
1357 return scan_operation(conn, namespace, args);
1358 };
1359
1360 if Instant::now() >= deadline {
1361 return Err(AppError::Timeout {
1362 operation: format!("enrich {:?} scan", args.operation()),
1363 duration_secs: 0,
1364 });
1365 }
1366
1367 let handle = conn.get_interrupt_handle();
1368 let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
1369 let stop_w = std::sync::Arc::clone(&stop);
1370 let watchdog = std::thread::spawn(move || {
1371 while !stop_w.load(std::sync::atomic::Ordering::Relaxed) {
1372 if Instant::now() >= deadline {
1373 handle.interrupt();
1374 break;
1375 }
1376 std::thread::sleep(std::time::Duration::from_millis(50));
1377 }
1378 });
1379
1380 let scan_t0 = Instant::now();
1381 let result = scan_operation(conn, namespace, args);
1382 stop.store(true, std::sync::atomic::Ordering::Relaxed);
1383 let _ = watchdog.join();
1384
1385 match result {
1386 Ok(v) => Ok(v),
1387 Err(AppError::Database(ref e)) if is_sqlite_interrupt(e) => Err(AppError::Timeout {
1388 operation: format!("enrich {:?} scan", args.operation()),
1389 duration_secs: scan_t0.elapsed().as_secs().max(1),
1390 }),
1391 Err(e) => Err(e),
1392 }
1393}
1394
1395fn is_sqlite_interrupt(err: &rusqlite::Error) -> bool {
1396 match err {
1397 rusqlite::Error::SqliteFailure(code, _) => {
1398 code.code == rusqlite::ErrorCode::OperationInterrupted || code.extended_code == 9
1399 }
1401 other => {
1402 let s = other.to_string().to_ascii_lowercase();
1403 s.contains("interrupt") || s.contains("cancelled")
1404 }
1405 }
1406}
1407
1408pub fn run(
1409 args: &EnrichArgs,
1410 llm_backend: crate::cli::LlmBackendChoice,
1411 embedding_backend: crate::cli::EmbeddingBackendChoice,
1412) -> Result<(), AppError> {
1413 validate_mode_conditional_flags_enrich(args)?;
1416
1417 if args.target != ReEmbedTarget::Memories
1420 && !matches!(args.operation(), EnrichOperation::ReEmbed)
1421 {
1422 let target_label = match args.target {
1423 ReEmbedTarget::Memories => "memories",
1424 ReEmbedTarget::Entities => "entities",
1425 ReEmbedTarget::Chunks => "chunks",
1426 ReEmbedTarget::All => "all",
1427 };
1428 return Err(AppError::Validation(format!(
1429 "--target {target_label} only applies to --operation re-embed"
1430 )));
1431 }
1432
1433 if args.list_dead
1442 || args.requeue_dead
1443 || args.prune_dead_orphans
1444 || args.prune_dead_entity_orphans
1445 {
1446 let namespace = crate::namespace::resolve_namespace(args.namespace.as_deref())?;
1447 let op_label = format!("{:?}", args.operation());
1448 let paths = AppPaths::resolve(args.db.as_deref())?;
1449 let queue_path = crate::paths::sidecar_path(&paths.db, ".enrich-queue.sqlite");
1450 let queue_conn = open_queue_db(&queue_path)?;
1451 if args.prune_dead_entity_orphans {
1456 let pruned = prune_dead_entity_orphans(&queue_conn, &op_label)?;
1457 let dead_total: i64 = queue_conn
1458 .query_row(
1459 "SELECT COUNT(*) FROM queue WHERE status='dead' \
1460 AND item_type='entity' \
1461 AND (operation = ?1 OR operation IS NULL)",
1462 rusqlite::params![op_label],
1463 |r| r.get(0),
1464 )
1465 .unwrap_or(0);
1466 emit_json(&DeadSummary {
1467 summary: true,
1468 operation: op_label,
1469 namespace,
1470 action: "prune-dead-entity-orphans",
1471 dead_total,
1472 requeued: 0,
1473 pruned,
1474 });
1475 return Ok(());
1476 }
1477 if args.prune_dead_orphans {
1480 ensure_db_ready(&paths)?;
1481 let main_conn = open_rw(&paths.db)?;
1482 let pruned = prune_dead_orphans(&queue_conn, &main_conn, &op_label, &namespace)?;
1483 let dead_total: i64 = queue_conn
1484 .query_row(
1485 "SELECT COUNT(*) FROM queue WHERE status='dead' \
1486 AND (operation = ?1 OR operation IS NULL)",
1487 rusqlite::params![op_label],
1488 |r| r.get(0),
1489 )
1490 .unwrap_or(0);
1491 emit_json(&DeadSummary {
1492 summary: true,
1493 operation: op_label,
1494 namespace,
1495 action: "prune-dead-orphans",
1496 dead_total,
1497 requeued: 0,
1498 pruned,
1499 });
1500 return Ok(());
1501 }
1502 if args.list_dead {
1503 let mut stmt = queue_conn.prepare(
1504 "SELECT item_key, item_type, attempt, error_class, error, \
1505 finish_reason, input_tokens, output_tokens FROM queue \
1506 WHERE status='dead' AND (operation = ?1 OR operation IS NULL) ORDER BY id",
1507 )?;
1508 let rows = stmt
1509 .query_map(rusqlite::params![op_label], |r| {
1510 Ok(DeadItem {
1511 dead_item: true,
1512 item_key: r.get(0)?,
1513 item_type: r.get(1)?,
1514 attempt: r.get(2)?,
1515 error_class: r.get(3)?,
1516 error: r.get(4)?,
1517 finish_reason: r.get(5)?,
1518 input_tokens: r.get(6)?,
1519 output_tokens: r.get(7)?,
1520 })
1521 })?
1522 .collect::<Result<Vec<_>, _>>()?;
1523 let dead_total = rows.len() as i64;
1524 for item in &rows {
1525 emit_json(item);
1526 }
1527 emit_json(&DeadSummary {
1528 summary: true,
1529 operation: op_label,
1530 namespace,
1531 action: "list-dead",
1532 dead_total,
1533 requeued: 0,
1534 pruned: 0,
1535 });
1536 return Ok(());
1537 }
1538 let dead_total: i64 = queue_conn
1540 .query_row(
1541 "SELECT COUNT(*) FROM queue WHERE status='dead' \
1542 AND (operation = ?1 OR operation IS NULL)",
1543 rusqlite::params![op_label],
1544 |r| r.get(0),
1545 )
1546 .unwrap_or(0);
1547 let requeued = queue_conn
1548 .execute(
1549 "UPDATE queue SET status='pending', attempt=0, next_retry_at=NULL, \
1550 error=NULL, error_class=NULL \
1551 WHERE status='dead' AND (operation = ?1 OR operation IS NULL)",
1552 rusqlite::params![op_label],
1553 )
1554 .map_err(|e| AppError::Validation(format!("requeue-dead failed: {e}")))?
1555 as i64;
1556 let _ = queue_conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);");
1557 emit_json(&DeadSummary {
1558 summary: true,
1559 operation: op_label,
1560 namespace,
1561 action: "requeue-dead",
1562 dead_total,
1563 requeued,
1564 pruned: 0,
1565 });
1566 return Ok(());
1567 }
1568
1569 if args.status {
1570 let paths = AppPaths::resolve(args.db.as_deref())?;
1571 ensure_db_ready(&paths)?;
1572 let conn = open_rw(&paths.db)?;
1573 let namespace = crate::namespace::resolve_namespace(args.namespace.as_deref())?;
1574 let unbound_backlog = scan_unbound_memories(&conn, &namespace, None, &[])?.len();
1575 let scan_backlog =
1578 count_operation_backlog(&conn, &args.operation(), &namespace, args.target)?;
1579 let queue_path = crate::paths::sidecar_path(&paths.db, ".enrich-queue.sqlite");
1580 let queue_conn = open_queue_db(&queue_path)?;
1581 let op_label = format!("{:?}", args.operation());
1582 let count_status = |st: &str, op: &str| -> i64 {
1586 queue_conn
1587 .query_row(
1588 "SELECT COUNT(*) FROM queue WHERE status=?1 \
1589 AND (operation = ?2 OR operation IS NULL)",
1590 rusqlite::params![st, op],
1591 |r| r.get(0),
1592 )
1593 .unwrap_or(0)
1594 };
1595 let eligible_now: i64 = queue_conn
1596 .query_row(
1597 "SELECT COUNT(*) FROM queue WHERE status='pending' \
1598 AND (operation = ?1 OR operation IS NULL) \
1599 AND (next_retry_at IS NULL OR next_retry_at <= datetime('now'))",
1600 rusqlite::params![op_label],
1601 |r| r.get(0),
1602 )
1603 .unwrap_or(0);
1604 let waiting: i64 = queue_conn
1605 .query_row(
1606 "SELECT COUNT(*) FROM queue WHERE status='pending' \
1607 AND (operation = ?1 OR operation IS NULL) \
1608 AND next_retry_at IS NOT NULL AND next_retry_at > datetime('now')",
1609 rusqlite::params![op_label],
1610 |r| r.get(0),
1611 )
1612 .unwrap_or(0);
1613 let waiting_items = {
1615 let mut stmt = queue_conn.prepare(
1616 "SELECT item_key, attempt, next_retry_at, error_class FROM queue \
1617 WHERE status='pending' AND (operation = ?1 OR operation IS NULL) \
1618 AND next_retry_at IS NOT NULL AND next_retry_at > datetime('now') \
1619 ORDER BY next_retry_at",
1620 )?;
1621 let items: Vec<WaitingItem> = stmt
1622 .query_map(rusqlite::params![op_label], |r| {
1623 Ok(WaitingItem {
1624 item_key: r.get(0)?,
1625 attempt: r.get(1)?,
1626 next_retry_at: r.get(2)?,
1627 error_class: r.get(3)?,
1628 })
1629 })?
1630 .collect::<Result<Vec<_>, _>>()?;
1631 items
1632 };
1633 let queue_pending = count_status("pending", &op_label);
1634 let queue_processing = count_status("processing", &op_label);
1635 let queue_done = count_status("done", &op_label);
1636 let queue_failed = count_status("failed", &op_label);
1637 let queue_skipped = count_status("skipped", &op_label);
1638 let queue_dead = count_status("dead", &op_label);
1639 let state = if eligible_now > 0 {
1641 "draining"
1642 } else if waiting > 0 {
1643 "cooldown"
1644 } else if queue_pending == 0 && scan_backlog > 0 {
1645 "pending-scan"
1646 } else {
1647 "empty"
1648 };
1649 emit_json(&EnrichStatus {
1650 status_report: true,
1651 operation: op_label,
1652 namespace,
1653 unbound_backlog,
1654 scan_backlog,
1655 queue_pending,
1656 queue_processing,
1657 queue_done,
1658 queue_failed,
1659 queue_skipped,
1660 queue_dead,
1661 eligible_now,
1662 waiting,
1663 state,
1664 waiting_items,
1665 });
1666 return Ok(());
1667 }
1668
1669 if args.reset_stale_claims {
1674 let paths = AppPaths::resolve(args.db.as_deref())?;
1675 ensure_db_ready(&paths)?;
1676 let queue_path = crate::paths::sidecar_path(&paths.db, ".enrich-queue.sqlite");
1677 let queue_conn = open_queue_db(&queue_path)?;
1678 let reset = reset_stale_processing_claims(&queue_conn, args.stale_claim_secs)?;
1679 let _ = queue_conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);");
1680 tracing::info!(
1681 target: "enrich",
1682 reset,
1683 max_age_secs = args.stale_claim_secs,
1684 "reset stale processing claims"
1685 );
1686 emit_json(&serde_json::json!({
1687 "reset_stale_claims": true,
1688 "reset": reset,
1689 "max_age_secs": args.stale_claim_secs,
1690 }));
1691 return Ok(());
1692 }
1693
1694 if args.mode() == EnrichMode::OpenRouter {
1699 let model = args.openrouter_model.as_deref().ok_or_else(|| {
1700 AppError::Validation(
1701 "--mode openrouter requires --openrouter-model (no default model is allowed)"
1702 .into(),
1703 )
1704 })?;
1705 let resolved =
1706 crate::config::resolve_api_key("openrouter", args.openrouter_api_key.as_deref())
1707 .ok_or_else(|| {
1708 AppError::Validation(
1709 "OPENROUTER_API_KEY not found; set the env var, store it via \
1710 `config add-key --provider openrouter`, or pass --openrouter-api-key"
1711 .into(),
1712 )
1713 })?;
1714 crate::embedder::get_openrouter_chat_client(
1715 resolved.value,
1716 model,
1717 args.openrouter_timeout,
1718 )?;
1719 }
1720
1721 let started = Instant::now();
1722
1723 let paths = AppPaths::resolve(args.db.as_deref())?;
1724 ensure_db_ready(&paths)?;
1725 let conn = open_rw(&paths.db)?;
1726 let namespace = crate::namespace::resolve_namespace(args.namespace.as_deref())?;
1727
1728 let wait_secs = args.wait_job_singleton;
1734 let force_flag = args.force_job_singleton;
1735 let _singleton = crate::lock::acquire_job_singleton(
1736 crate::lock::JobType::Enrich,
1737 &namespace,
1738 &paths.db,
1739 wait_secs,
1740 force_flag,
1741 )?;
1742
1743 let provider_binary = if matches!(args.operation(), EnrichOperation::ReEmbed) {
1745 None
1746 } else {
1747 Some(match args.mode() {
1748 EnrichMode::ClaudeCode => {
1749 let bin = find_claude_binary(args.claude_binary.as_deref())?;
1750 let version = super::claude_runner::validate_claude_version(&bin)?;
1751 tracing::info!(target: "enrich", binary = %bin.display(), version = %version, "Claude Code binary validated");
1752 emit_json(&PhaseEvent {
1753 phase: "validate",
1754 binary_path: bin.to_str(),
1755 version: Some(&version),
1756 items_total: None,
1757 items_pending: None,
1758 llm_parallelism: None,
1759 });
1760 bin
1761 }
1762 EnrichMode::Codex => {
1763 let bin = find_codex_binary(args.codex_binary.as_deref())?;
1764 emit_json(&PhaseEvent {
1765 phase: "validate",
1766 binary_path: bin.to_str(),
1767 version: None,
1768 items_total: None,
1769 items_pending: None,
1770 llm_parallelism: None,
1771 });
1772 bin
1773 }
1774 EnrichMode::Opencode => {
1775 let bin = super::opencode_runner::find_opencode_binary_with_override(
1776 args.opencode_binary.as_deref(),
1777 )?;
1778 emit_json(&PhaseEvent {
1779 phase: "validate",
1780 binary_path: bin.to_str(),
1781 version: None,
1782 items_total: None,
1783 items_pending: None,
1784 llm_parallelism: None,
1785 });
1786 bin
1787 }
1788 EnrichMode::OpenRouter => {
1789 emit_json(&PhaseEvent {
1794 phase: "validate",
1795 binary_path: None,
1796 version: None,
1797 items_total: None,
1798 items_pending: None,
1799 llm_parallelism: None,
1800 });
1801 PathBuf::new()
1802 }
1803 })
1804 };
1805
1806 if args.max_load_check && !args.dry_run && crate::system_load::is_system_saturated() {
1810 let load = crate::system_load::load_average_one();
1811 let n = crate::system_load::ncpus();
1812 return Err(AppError::Validation(format!(
1813 "system load average {load:.2} exceeds 2x ncpus ({n}); \
1814 pass --no-max-load-check to override (not recommended)"
1815 )));
1816 }
1817
1818 if args.preflight_check
1825 && !args.dry_run
1826 && !matches!(args.operation(), EnrichOperation::ReEmbed)
1827 {
1828 let preflight_result = run_preflight_probe(args);
1829 match preflight_result {
1830 PreflightOutcome::Healthy => {
1831 tracing::info!(target: "enrich", mode = ?args.mode(), "preflight probe healthy");
1832 }
1833 PreflightOutcome::RateLimited { reason, suggestion } => {
1834 if let Some(fallback) = args.fallback_mode.clone() {
1835 if fallback != args.mode() {
1836 return Err(AppError::Validation(format!(
1846 "preflight detected rate limit on {mode:?}: {reason}; \
1847 re-invoke with `--mode {fallback:?}` to use the fallback provider",
1848 mode = args.mode()
1849 )));
1850 }
1851 return Err(AppError::Validation(format!(
1852 "preflight detected rate limit on {mode:?}: {reason}; \
1853 --fallback-mode matches --mode, no recovery possible",
1854 mode = args.mode()
1855 )));
1856 }
1857 return Err(AppError::Validation(format!(
1858 "preflight detected rate limit on {mode:?}: {reason}; \
1859 {suggestion}; pass --fallback-mode codex to recover",
1860 mode = args.mode()
1861 )));
1862 }
1863 PreflightOutcome::Error(e) => {
1864 return Err(e);
1865 }
1866 }
1867 }
1868
1869 let max_runtime_secs = args.max_runtime.unwrap_or(3600);
1875 let until_deadline = Instant::now() + std::time::Duration::from_secs(max_runtime_secs);
1876 let pair_scan_ops = matches!(
1877 args.operation(),
1878 EnrichOperation::EntityConnect | EnrichOperation::CrossDomainBridges
1879 );
1880 const ENTITY_CONNECT_SCAN_SOFT_CEILING_SECS: u64 = 120;
1882 let scan_deadline = if pair_scan_ops {
1883 let soft =
1884 Instant::now() + std::time::Duration::from_secs(ENTITY_CONNECT_SCAN_SOFT_CEILING_SECS);
1885 Some(soft.min(until_deadline))
1886 } else if args.until_empty || args.max_runtime.is_some() {
1887 Some(until_deadline)
1888 } else {
1889 None
1890 };
1891
1892 let pair_op_cli = enrich_operation_cli_name(&args.operation());
1893 let mut backlog_degree0_proxy: Option<i64> = None;
1894 if pair_scan_ops {
1895 let entities_in_namespace: i64 = conn
1896 .query_row(
1897 "SELECT COUNT(*) FROM entities WHERE namespace = ?1",
1898 rusqlite::params![namespace],
1899 |r| r.get(0),
1900 )
1901 .unwrap_or(0);
1902 backlog_degree0_proxy =
1904 count_operation_backlog(&conn, &args.operation(), &namespace, args.target).ok();
1905 emit_json(&ScanStartEvent {
1906 phase: "scan_start",
1907 operation: pair_op_cli,
1908 entities_in_namespace,
1909 backlog_degree0_proxy,
1910 pair_algorithm: Some("cooccurrence+hub_island"),
1911 limit: args.limit,
1912 scan_deadline_secs: scan_deadline
1913 .map(|d| d.saturating_duration_since(Instant::now()).as_secs()),
1914 });
1915 }
1916
1917 let scan_started = Instant::now();
1919 let mut scan_result = scan_operation_with_deadline(&conn, &namespace, args, scan_deadline)?;
1920 if matches!(args.operation(), EnrichOperation::BodyEnrich) {
1929 let q_path = crate::paths::sidecar_path(&paths.db, ".enrich-queue.sqlite");
1930 if let Ok(q) = open_queue_db(&q_path) {
1931 if let Ok(vetoed) = skipped_item_keys(&q, &format!("{:?}", args.operation())) {
1932 scan_result.retain(|k| !vetoed.contains(k));
1933 }
1934 }
1935 }
1936 let total = scan_result.len();
1937 let scan_elapsed_ms = scan_started.elapsed().as_millis() as u64;
1938
1939 emit_json(&PhaseEvent {
1940 phase: "scan",
1941 binary_path: None,
1942 version: None,
1943 items_total: Some(total),
1944 items_pending: Some(total),
1945 llm_parallelism: Some(args.llm_parallelism),
1946 });
1947 if pair_scan_ops {
1948 emit_json(&serde_json::json!({
1949 "phase": "scan_meta",
1950 "operation": pair_op_cli,
1951 "pair_algorithm": "cooccurrence+hub_island",
1952 "items_total": total,
1953 "pairs_enqueued_this_scan": total,
1954 "backlog_degree0_proxy": backlog_degree0_proxy,
1955 "scan_elapsed_ms": scan_elapsed_ms,
1956 "scan_aborted_reason": serde_json::Value::Null,
1957 }));
1958 }
1959
1960 if args.dry_run {
1962 for (idx, key) in scan_result.iter().enumerate() {
1963 emit_json(&ItemEvent {
1964 item: key,
1965 status: "preview",
1966 memory_id: None,
1967 entity_id: None,
1968 entities: None,
1969 rels: None,
1970 chars_before: None,
1971 chars_after: None,
1972 cost_usd: None,
1973 elapsed_ms: None,
1974 error: None,
1975 index: idx,
1976 total,
1977 });
1978 }
1979 emit_json(&EnrichSummary {
1980 summary: true,
1981 operation: format!("{:?}", args.operation()),
1982 items_total: total,
1983 completed: 0,
1984 failed: 0,
1985 skipped: 0,
1986 cost_usd: 0.0,
1987 elapsed_ms: started.elapsed().as_millis() as u64,
1988 backend_invoked: take_enrich_backend(),
1989 waiting: 0,
1990 dead: 0,
1991 });
1992 return Ok(());
1993 }
1994
1995 let queue_path = crate::paths::sidecar_path(&paths.db, ".enrich-queue.sqlite");
2001 let mut queue_conn = open_queue_db(&queue_path)?;
2002
2003 {
2008 let stale_reset = reset_stale_processing_claims(&queue_conn, args.stale_claim_secs)?;
2009 if stale_reset > 0 {
2010 tracing::info!(
2011 target: "enrich",
2012 count = stale_reset,
2013 max_age_secs = args.stale_claim_secs,
2014 "reset stale processing claims (older than threshold)"
2015 );
2016 }
2017 }
2018
2019 if args.resume {
2020 let reset = queue_conn
2021 .execute(
2022 "UPDATE queue SET status='pending' WHERE status='processing'",
2023 [],
2024 )
2025 .map_err(|e| AppError::Validation(format!("queue resume failed: {e}")))?;
2026 if reset > 0 {
2027 tracing::info!(target: "enrich", count = reset, "reset stuck processing items to pending");
2028 }
2029 }
2030
2031 if args.retry_failed {
2032 let count = queue_conn
2033 .execute(
2034 "UPDATE queue SET status='pending', attempt=0 WHERE status='failed'",
2035 [],
2036 )
2037 .map_err(|e| AppError::Validation(format!("queue retry-failed reset failed: {e}")))?;
2038 tracing::info!(target: "enrich", count, "retrying failed items");
2039 }
2040
2041 if !args.resume && !args.retry_failed && !args.until_empty {
2042 queue_conn
2043 .execute("DELETE FROM queue", [])
2044 .map_err(|e| AppError::Validation(format!("queue clear failed: {e}")))?;
2045 }
2046
2047 let op_label = format!("{:?}", args.operation());
2053 let item_type = item_type_for(&args.operation());
2054 {
2055 let tx = queue_conn.transaction()?;
2056 let tx_conn: &Connection = &tx;
2059 for key in scan_result.iter() {
2060 let it = item_type_for_key(key, item_type);
2064 enqueue_candidate(tx_conn, &conn, &namespace, key, it, &op_label);
2065 }
2066 tx.commit()?;
2067 }
2068
2069 let parallelism = if args.mode() == EnrichMode::OpenRouter {
2072 let rest = args.rest_concurrency.unwrap_or(8).clamp(1, 16) as usize;
2073 tracing::info!(
2074 target: "enrich",
2075 concurrency = rest,
2076 source = "rest_concurrency",
2077 "OpenRouter REST concurrency (clamp 1..=16)"
2078 );
2079 rest
2080 } else {
2081 let p = args.llm_parallelism.clamp(1, 32) as usize;
2082 tracing::info!(
2083 target: "enrich",
2084 concurrency = p,
2085 source = "llm_parallelism",
2086 "LLM subprocess parallelism (clamp 1..=32)"
2087 );
2088 p
2089 };
2090 if parallelism > 1 {
2091 tracing::info!(
2092 target: "enrich",
2093 llm_parallelism = parallelism,
2094 "parallel LLM processing with bounded thread pool"
2095 );
2096 }
2097 if parallelism > 4 {
2101 match args.mode() {
2102 EnrichMode::ClaudeCode => {
2103 tracing::warn!(
2104 target: "enrich",
2105 llm_parallelism = parallelism,
2106 recommended_max = 4,
2107 mode = "claude-code",
2108 "llm_parallelism above 4 multiplies Claude Code subprocess fan-out; \
2109 consider combining with SQLITE_GRAPHRAG_CLAUDE_EMPTY_CONFIG_DIR \
2110 to cut MCP children (G28-A)"
2111 );
2112 }
2113 EnrichMode::Codex if parallelism > 16 => {
2114 tracing::warn!(
2115 target: "enrich",
2116 llm_parallelism = parallelism,
2117 recommended_max = 16,
2118 mode = "codex",
2119 "llm_parallelism above 16 risks OAuth rate-limit on Codex; \
2120 consider --llm-parallelism 8 for safer concurrency"
2121 );
2122 }
2123 EnrichMode::Codex => {
2124 }
2128 EnrichMode::Opencode if parallelism > 16 => {
2129 tracing::warn!(
2130 target: "enrich",
2131 llm_parallelism = parallelism,
2132 recommended_max = 16,
2133 mode = "opencode",
2134 "llm_parallelism above 16 risks OAuth rate-limit on OpenCode; \
2135 consider --llm-parallelism 8 for safer concurrency"
2136 );
2137 }
2138 EnrichMode::Opencode => {
2139 }
2141 EnrichMode::OpenRouter => {
2142 }
2145 }
2146 }
2147
2148 let mut completed = 0usize;
2149 let mut failed = 0usize;
2150 let mut skipped = 0usize;
2151 let mut cost_total = 0.0f64;
2152 let mut oauth_detected = false;
2153 let mut backoff_secs = DEFAULT_RATE_LIMIT_WAIT;
2154 let rate_limit_deadline = std::time::Instant::now() + std::time::Duration::from_secs(3600);
2155 let enrich_started = std::time::Instant::now();
2156
2157 let provider_timeout = match args.mode() {
2158 EnrichMode::ClaudeCode => args.claude_timeout,
2159 EnrichMode::Codex => args.codex_timeout,
2160 EnrichMode::Opencode => args.opencode_timeout,
2161 EnrichMode::OpenRouter => args.openrouter_timeout,
2162 };
2163
2164 let provider_model: Option<&str> = match args.mode() {
2165 EnrichMode::ClaudeCode => args.claude_model.as_deref(),
2166 EnrichMode::Codex => args.codex_model.as_deref(),
2167 EnrichMode::Opencode => args.opencode_model.as_deref(),
2168 EnrichMode::OpenRouter => args.openrouter_model.as_deref(),
2169 };
2170
2171 let backoff_clause: &str = if args.ignore_backoff {
2175 ""
2176 } else {
2177 "AND (next_retry_at IS NULL OR next_retry_at <= datetime('now'))"
2178 };
2179
2180 emit_json(&ConcurrencyEvent {
2183 phase: "concurrency",
2184 scan_parallelism: 1,
2185 drain_parallelism: parallelism as u32,
2186 });
2187
2188 let mut until_empty_iter: u32 = 0;
2196 loop {
2197 if args.until_empty {
2198 until_empty_iter = until_empty_iter.saturating_add(1);
2199 if until_empty_iter > 1 {
2200 let mut rescan =
2204 scan_operation_with_deadline(&conn, &namespace, args, Some(until_deadline))?;
2205 if matches!(args.operation(), EnrichOperation::BodyEnrich) {
2210 if let Ok(vetoed) = skipped_item_keys(&queue_conn, &op_label) {
2211 rescan.retain(|k| !vetoed.contains(k));
2212 }
2213 }
2214 {
2216 let tx = queue_conn.transaction()?;
2217 let tx_conn: &Connection = &tx;
2218 for key in &rescan {
2219 let it = item_type_for_key(key, item_type);
2220 enqueue_candidate(tx_conn, &conn, &namespace, key, it, &op_label);
2221 }
2222 tx.commit()?;
2223 }
2224 }
2225 }
2226 let completed_before = completed;
2227
2228 if parallelism > 1 {
2232 let stdout_mu = parking_lot::Mutex::new(());
2233 let budget = args.max_cost_usd;
2234 let operation = args.operation().clone();
2235 let mode = args.mode().clone();
2236 let min_oc = args.min_output_chars;
2237 let max_oc = args.max_output_chars;
2238 let prompt_tpl = args.prompt_template.as_deref().map(|p| p.to_path_buf());
2239
2240 struct WorkerResult {
2241 completed: usize,
2242 failed: usize,
2243 skipped: usize,
2244 cost: f64,
2245 oauth: bool,
2246 db_busy: bool,
2251 }
2252
2253 let results: Vec<WorkerResult> = std::thread::scope(|s| {
2254 let handles: Vec<_> = (0..parallelism)
2255 .map(|worker_id| {
2256 let stdout_mu = &stdout_mu;
2257 let paths = &paths;
2258 let queue_path = &queue_path;
2259 let namespace = &namespace;
2260 let provider_binary = provider_binary.as_deref();
2261 let operation = &operation;
2262 let mode = &mode;
2263 let prompt_tpl = prompt_tpl.as_deref();
2264 s.spawn(move || {
2265 let w_conn = match open_rw(&paths.db) {
2266 Ok(c) => c,
2267 Err(e) => {
2268 tracing::error!(target: "enrich", worker = worker_id, error = %e, "worker failed to open DB");
2269 return WorkerResult { completed: 0, failed: 0, skipped: 0, cost: 0.0, oauth: false, db_busy: false };
2270 }
2271 };
2272 let w_queue = match open_queue_db(queue_path) {
2273 Ok(c) => c,
2274 Err(e) => {
2275 tracing::error!(target: "enrich", worker = worker_id, error = %e, "worker failed to open queue DB");
2276 return WorkerResult { completed: 0, failed: 0, skipped: 0, cost: 0.0, oauth: false, db_busy: false };
2277 }
2278 };
2279 let mut w_completed = 0usize;
2280 let mut w_failed = 0usize;
2281 let mut w_skipped = 0usize;
2282 let mut w_cost = 0.0f64;
2283 let mut w_oauth = false;
2284 let mut w_db_busy = false;
2285 let mut w_backoff = DEFAULT_RATE_LIMIT_WAIT;
2286 let w_deadline = std::time::Instant::now() + std::time::Duration::from_secs(3600);
2287 let mut w_breaker = crate::retry::CircuitBreaker::new(
2293 args.circuit_breaker_threshold.max(1),
2294 std::time::Duration::from_secs(60),
2295 );
2296
2297 loop {
2298 if crate::shutdown_requested() {
2299 tracing::info!(target: "enrich", "shutdown requested, worker stopping");
2300 break;
2301 }
2302 if let Some(b) = budget {
2303 if !w_oauth && w_cost >= b {
2304 break;
2305 }
2306 }
2307 let pending = match crate::storage::utils::with_busy_retry(|| {
2326 dequeue_next_pending(&w_queue, backoff_clause)
2327 }) {
2328 Ok(DequeueOutcome::Claimed(p)) => Some(p),
2329 Ok(DequeueOutcome::Empty) => None,
2330 Err(AppError::DbBusy(msg)) => {
2331 tracing::error!(target: "enrich", worker = worker_id, error = %msg, "SQLITE_BUSY exhausted bounded retries, worker aborting");
2332 w_db_busy = true;
2333 None
2334 }
2335 Err(e) => {
2336 tracing::error!(target: "enrich", worker = worker_id, error = %e, "dequeue failed");
2337 None
2338 }
2339 };
2340 let (queue_id, item_key, _item_type, attempt_current) = match pending {
2341 Some(p) => p,
2342 None => break,
2343 };
2344 let _ = heartbeat(&w_queue, queue_id);
2349 let item_started = Instant::now();
2350 let current_index = w_completed + w_failed + w_skipped;
2351
2352 let provider_bin = provider_binary.unwrap_or_else(|| std::path::Path::new(""));
2358 let call_result = match operation {
2359 EnrichOperation::MemoryBindings | EnrichOperation::AugmentBindings => call_memory_bindings(&w_conn, namespace, &item_key, provider_bin, provider_model, provider_timeout, mode),
2360 EnrichOperation::EntityDescriptions => call_entity_description(&w_conn, namespace, &item_key, provider_bin, provider_model, provider_timeout, mode),
2361 EnrichOperation::BodyEnrich => call_body_enrich(&w_conn, namespace, &item_key, provider_bin, provider_model, provider_timeout, mode, min_oc, max_oc, prompt_tpl, args.preserve_threshold, paths, llm_backend, embedding_backend),
2362 EnrichOperation::ReEmbed => call_reembed(&w_conn, namespace, &item_key, paths, llm_backend, embedding_backend),
2363 EnrichOperation::WeightCalibrate => call_weight_calibrate(&w_conn, namespace, &item_key, provider_bin, provider_model, provider_timeout, mode),
2364 EnrichOperation::RelationReclassify => call_relation_reclassify(&w_conn, namespace, &item_key, provider_bin, provider_model, provider_timeout, mode),
2365 EnrichOperation::EntityConnect | EnrichOperation::CrossDomainBridges => call_entity_connect(&w_conn, namespace, &item_key, provider_bin, provider_model, provider_timeout, mode),
2366 EnrichOperation::EntityTypeValidate => call_entity_type_validate(&w_conn, namespace, &item_key, provider_bin, provider_model, provider_timeout, mode),
2367 EnrichOperation::DescriptionEnrich => call_description_enrich(&w_conn, namespace, &item_key, provider_bin, provider_model, provider_timeout, mode),
2368 EnrichOperation::DomainClassify => call_domain_classify(&w_conn, namespace, &item_key, provider_bin, provider_model, provider_timeout, mode),
2369 EnrichOperation::GraphAudit => call_graph_audit(&w_conn, namespace, &item_key, provider_bin, provider_model, provider_timeout, mode),
2370 EnrichOperation::DeepResearchSynth => call_deep_research_synth(&w_conn, namespace, &item_key, provider_bin, provider_model, provider_timeout, mode),
2371 EnrichOperation::BodyExtract => call_body_extract(&w_conn, namespace, &item_key, provider_bin, provider_model, provider_timeout, mode, args.body_extract_graph_only),
2372 };
2373 let openrouter_diag = take_last_openrouter_failure();
2379
2380 match call_result {
2381 Ok(EnrichItemResult::Done { cost, is_oauth, memory_id, entity_id, entities, rels, chars_before, chars_after }) => {
2382 if is_oauth { w_oauth = true; }
2383 w_backoff = DEFAULT_RATE_LIMIT_WAIT;
2384 let _ = w_queue.execute(
2385 "UPDATE queue SET status='done', memory_id=?1, entity_id=?2, entities=?3, rels=?4, cost_usd=?5, elapsed_ms=?6, done_at=datetime('now') WHERE id=?7",
2386 rusqlite::params![memory_id, entity_id, entities as i64, rels as i64, cost, item_started.elapsed().as_millis() as i64, queue_id],
2387 );
2388 w_completed += 1;
2389 if !is_oauth { w_cost += cost; }
2390 let _ = w_breaker
2392 .record(crate::retry::AttemptOutcome::Success);
2393 let _guard = stdout_mu.lock();
2394 emit_json(&ItemEvent { item: &item_key, status: "done", memory_id, entity_id, entities: Some(entities), rels: Some(rels), chars_before, chars_after, cost_usd: if is_oauth { None } else { Some(cost) }, elapsed_ms: Some(item_started.elapsed().as_millis() as u64), error: None, index: current_index, total });
2395 }
2396 Ok(EnrichItemResult::Skipped { reason }) => {
2397 w_skipped += 1;
2398 let _ = w_queue.execute("UPDATE queue SET status='skipped', error=?1, done_at=datetime('now') WHERE id=?2", rusqlite::params![reason, queue_id]);
2399 let _guard = stdout_mu.lock();
2400 emit_json(&ItemEvent { item: &item_key, status: "skipped", memory_id: None, entity_id: None, entities: None, rels: None, chars_before: None, chars_after: None, cost_usd: None, elapsed_ms: Some(item_started.elapsed().as_millis() as u64), error: None, index: current_index, total });
2401 }
2402 Ok(EnrichItemResult::PreservationFailed { score, threshold, chars_before, chars_after }) => {
2403 w_skipped += 1;
2409 let reason = format!(
2410 "preservation_failed: jaccard={score:.3} threshold={threshold:.3} (orig={chars_before} chars, new={chars_after} chars)"
2411 );
2412 let _ = w_queue.execute(
2413 "UPDATE queue SET status='skipped', error=?1, done_at=datetime('now') WHERE id=?2",
2414 rusqlite::params![reason, queue_id],
2415 );
2416 let _guard = stdout_mu.lock();
2417 emit_json(&ItemEvent {
2418 item: &item_key,
2419 status: "preservation_failed",
2420 memory_id: None,
2421 entity_id: None,
2422 entities: None,
2423 rels: None,
2424 chars_before: Some(chars_before),
2425 chars_after: Some(chars_after),
2426 cost_usd: None,
2427 elapsed_ms: Some(item_started.elapsed().as_millis() as u64),
2428 error: Some(reason),
2429 index: current_index,
2430 total,
2431 });
2432 }
2433 Err(e) => {
2434 let err_str = format!("{e}");
2435 if matches!(e, AppError::RateLimited { .. }) {
2436 if crate::retry::is_kill_switch_active() {
2437 tracing::warn!(target: "enrich", "SQLITE_GRAPHRAG_DISABLE_RETRY=1, skipping rate-limit retry");
2438 } else if std::time::Instant::now() >= w_deadline {
2439 tracing::error!(target: "enrich", "rate-limit retry deadline (1h) exhausted in worker");
2440 } else {
2441 let half = w_backoff / 2;
2442 let jitter = if half == 0 { 0 } else { fastrand::u64(0..half) };
2443 let actual_wait = half + jitter;
2444 tracing::warn!(target: "enrich", delay_secs = actual_wait, error_kind = "rate_limited", "rate limited in worker, backing off");
2445 let _ = w_queue.execute("UPDATE queue SET status='pending' WHERE id=?1", rusqlite::params![queue_id]);
2446 std::thread::sleep(std::time::Duration::from_secs(actual_wait));
2447 w_backoff = (w_backoff * 2).min(900);
2448 continue;
2449 }
2450 }
2451 w_failed += 1;
2452 let outcome = match openrouter_diag {
2459 Some(diag) => record_item_failure_typed(
2460 &w_queue,
2461 queue_id,
2462 attempt_current,
2463 args.max_attempts,
2464 diag.retry_class,
2465 &err_str,
2466 diag.finish_reason.as_deref(),
2467 diag.prompt_tokens,
2468 diag.completion_tokens,
2469 ),
2470 None => record_item_failure(&w_queue, queue_id, attempt_current, args.max_attempts, &e),
2471 };
2472 let _guard = stdout_mu.lock();
2473 emit_json(&ItemEvent { item: &item_key, status: "failed", memory_id: None, entity_id: None, entities: None, rels: None, chars_before: None, chars_after: None, cost_usd: None, elapsed_ms: Some(item_started.elapsed().as_millis() as u64), error: Some(err_str), index: current_index, total });
2474 let breaker_opened = w_breaker.record(outcome);
2477 if breaker_opened {
2478 tracing::error!(target: "enrich",
2479 consecutive_failures = w_breaker.consecutive_failures(),
2480 "circuit breaker opened — aborting worker"
2481 );
2482 break;
2483 }
2484 }
2485 }
2486 }
2487 WorkerResult { completed: w_completed, failed: w_failed, skipped: w_skipped, cost: w_cost, oauth: w_oauth, db_busy: w_db_busy }
2488 })
2489 })
2490 .collect();
2491 handles
2492 .into_iter()
2493 .map(|h| {
2494 h.join().unwrap_or(WorkerResult {
2495 completed: 0,
2496 failed: 0,
2497 skipped: 0,
2498 cost: 0.0,
2499 oauth: false,
2500 db_busy: false,
2501 })
2502 })
2503 .collect()
2504 });
2505
2506 if results.iter().any(|r| r.db_busy) {
2512 return Err(AppError::DbBusy(
2513 "SQLITE_BUSY exhausted bounded retries while dequeuing (parallel worker)"
2514 .into(),
2515 ));
2516 }
2517
2518 for r in &results {
2519 completed += r.completed;
2520 failed += r.failed;
2521 skipped += r.skipped;
2522 cost_total += r.cost;
2523 if r.oauth && !oauth_detected {
2524 oauth_detected = true;
2525 }
2526 }
2527 } else {
2528 loop {
2530 if crate::shutdown_requested() {
2531 tracing::info!(target: "enrich", "shutdown requested, stopping enrichment");
2532 break;
2533 }
2534
2535 if let Some(budget) = args.max_cost_usd {
2537 if !oauth_detected && cost_total >= budget {
2538 tracing::warn!(target: "enrich", spent = cost_total, budget, "budget exceeded, stopping");
2539 break;
2540 }
2541 }
2542
2543 let pending = match crate::storage::utils::with_busy_retry(|| {
2559 dequeue_next_pending(&queue_conn, backoff_clause)
2560 }) {
2561 Ok(DequeueOutcome::Claimed(p)) => Some(p),
2562 Ok(DequeueOutcome::Empty) => None,
2563 Err(e @ AppError::DbBusy(_)) => {
2564 tracing::error!(target: "enrich", error = %e, "SQLITE_BUSY exhausted bounded retries, aborting drain loop");
2565 return Err(e);
2566 }
2567 Err(e) => {
2568 tracing::error!(target: "enrich", error = %e, "dequeue failed");
2569 None
2570 }
2571 };
2572
2573 let (queue_id, item_key, item_type, attempt_current) = match pending {
2574 Some(p) => p,
2575 None => break,
2576 };
2577
2578 let _ = heartbeat(&queue_conn, queue_id);
2581
2582 let item_started = Instant::now();
2583 let current_index = completed + failed + skipped;
2584
2585 let provider_bin = provider_binary
2588 .as_deref()
2589 .unwrap_or_else(|| std::path::Path::new(""));
2590 let call_result = match args.operation() {
2591 EnrichOperation::MemoryBindings | EnrichOperation::AugmentBindings => {
2592 call_memory_bindings(
2593 &conn,
2594 &namespace,
2595 &item_key,
2596 provider_bin,
2597 provider_model,
2598 provider_timeout,
2599 &args.mode(),
2600 )
2601 }
2602 EnrichOperation::EntityDescriptions => call_entity_description(
2603 &conn,
2604 &namespace,
2605 &item_key,
2606 provider_bin,
2607 provider_model,
2608 provider_timeout,
2609 &args.mode(),
2610 ),
2611 EnrichOperation::BodyEnrich => call_body_enrich(
2612 &conn,
2613 &namespace,
2614 &item_key,
2615 provider_bin,
2616 provider_model,
2617 provider_timeout,
2618 &args.mode(),
2619 args.min_output_chars,
2620 args.max_output_chars,
2621 args.prompt_template.as_deref(),
2622 args.preserve_threshold,
2623 &paths,
2624 llm_backend,
2625 embedding_backend,
2626 ),
2627 EnrichOperation::ReEmbed => call_reembed(
2628 &conn,
2629 &namespace,
2630 &item_key,
2631 &paths,
2632 llm_backend,
2633 embedding_backend,
2634 ),
2635 EnrichOperation::WeightCalibrate => call_weight_calibrate(
2636 &conn,
2637 &namespace,
2638 &item_key,
2639 provider_bin,
2640 provider_model,
2641 provider_timeout,
2642 &args.mode(),
2643 ),
2644 EnrichOperation::RelationReclassify => call_relation_reclassify(
2645 &conn,
2646 &namespace,
2647 &item_key,
2648 provider_bin,
2649 provider_model,
2650 provider_timeout,
2651 &args.mode(),
2652 ),
2653 EnrichOperation::EntityConnect | EnrichOperation::CrossDomainBridges => {
2654 call_entity_connect(
2655 &conn,
2656 &namespace,
2657 &item_key,
2658 provider_bin,
2659 provider_model,
2660 provider_timeout,
2661 &args.mode(),
2662 )
2663 }
2664 EnrichOperation::EntityTypeValidate => call_entity_type_validate(
2665 &conn,
2666 &namespace,
2667 &item_key,
2668 provider_bin,
2669 provider_model,
2670 provider_timeout,
2671 &args.mode(),
2672 ),
2673 EnrichOperation::DescriptionEnrich => call_description_enrich(
2674 &conn,
2675 &namespace,
2676 &item_key,
2677 provider_bin,
2678 provider_model,
2679 provider_timeout,
2680 &args.mode(),
2681 ),
2682 EnrichOperation::DomainClassify => call_domain_classify(
2683 &conn,
2684 &namespace,
2685 &item_key,
2686 provider_bin,
2687 provider_model,
2688 provider_timeout,
2689 &args.mode(),
2690 ),
2691 EnrichOperation::GraphAudit => call_graph_audit(
2692 &conn,
2693 &namespace,
2694 &item_key,
2695 provider_bin,
2696 provider_model,
2697 provider_timeout,
2698 &args.mode(),
2699 ),
2700 EnrichOperation::DeepResearchSynth => call_deep_research_synth(
2701 &conn,
2702 &namespace,
2703 &item_key,
2704 provider_bin,
2705 provider_model,
2706 provider_timeout,
2707 &args.mode(),
2708 ),
2709 EnrichOperation::BodyExtract => call_body_extract(
2710 &conn,
2711 &namespace,
2712 &item_key,
2713 provider_bin,
2714 provider_model,
2715 provider_timeout,
2716 &args.mode(),
2717 args.body_extract_graph_only,
2718 ),
2719 };
2720 let openrouter_diag = take_last_openrouter_failure();
2723
2724 match call_result {
2725 Ok(EnrichItemResult::Done {
2726 memory_id,
2727 entity_id,
2728 entities,
2729 rels,
2730 chars_before,
2731 chars_after,
2732 cost,
2733 is_oauth,
2734 }) => {
2735 if is_oauth && !oauth_detected {
2736 oauth_detected = true;
2737 tracing::info!(target: "enrich", "OAuth subscription detected — cost_usd omitted from output");
2738 }
2739 backoff_secs = DEFAULT_RATE_LIMIT_WAIT;
2740
2741 let persist_err: Option<String> = match args.operation() {
2743 EnrichOperation::MemoryBindings => {
2744 None
2746 }
2747 EnrichOperation::EntityDescriptions => {
2748 None
2750 }
2751 EnrichOperation::BodyEnrich => {
2752 None
2754 }
2755 _ => {
2756 None
2758 }
2759 };
2760
2761 if let Err(e) = queue_conn.execute(
2762 "UPDATE queue SET status='done', memory_id=?1, entity_id=?2, entities=?3, rels=?4, cost_usd=?5, elapsed_ms=?6, done_at=datetime('now') WHERE id=?7",
2763 rusqlite::params![
2764 memory_id,
2765 entity_id,
2766 entities as i64,
2767 rels as i64,
2768 cost,
2769 item_started.elapsed().as_millis() as i64,
2770 queue_id
2771 ],
2772 ) {
2773 tracing::warn!(target: "enrich", error = %e, "queue done update failed");
2774 }
2775
2776 if persist_err.is_none() {
2777 completed += 1;
2778 if !is_oauth {
2779 cost_total += cost;
2780 }
2781 emit_json(&ItemEvent {
2782 item: &item_key,
2783 status: "done",
2784 memory_id,
2785 entity_id,
2786 entities: Some(entities),
2787 rels: Some(rels),
2788 chars_before,
2789 chars_after,
2790 cost_usd: if is_oauth { None } else { Some(cost) },
2791 elapsed_ms: Some(item_started.elapsed().as_millis() as u64),
2792 error: None,
2793 index: current_index,
2794 total,
2795 });
2796 } else {
2797 failed += 1;
2798 emit_json(&ItemEvent {
2799 item: &item_key,
2800 status: "failed",
2801 memory_id: None,
2802 entity_id: None,
2803 entities: None,
2804 rels: None,
2805 chars_before: None,
2806 chars_after: None,
2807 cost_usd: None,
2808 elapsed_ms: Some(item_started.elapsed().as_millis() as u64),
2809 error: persist_err,
2810 index: current_index,
2811 total,
2812 });
2813 }
2814 }
2815 Ok(EnrichItemResult::Skipped { reason }) => {
2816 skipped += 1;
2817 if let Err(e) = queue_conn.execute(
2818 "UPDATE queue SET status='skipped', error=?1, done_at=datetime('now') WHERE id=?2",
2819 rusqlite::params![reason, queue_id],
2820 ) {
2821 tracing::warn!(target: "enrich", error = %e, "queue skipped update failed");
2822 }
2823 emit_json(&ItemEvent {
2824 item: &item_key,
2825 status: "skipped",
2826 memory_id: None,
2827 entity_id: None,
2828 entities: None,
2829 rels: None,
2830 chars_before: None,
2831 chars_after: None,
2832 cost_usd: None,
2833 elapsed_ms: Some(item_started.elapsed().as_millis() as u64),
2834 error: None,
2835 index: current_index,
2836 total,
2837 });
2838 }
2839 Ok(EnrichItemResult::PreservationFailed {
2840 score,
2841 threshold,
2842 chars_before,
2843 chars_after,
2844 }) => {
2845 skipped += 1;
2852 let reason = format!(
2853 "preservation_failed: jaccard={score:.3} threshold={threshold:.3} (orig={chars_before} chars, new={chars_after} chars)"
2854 );
2855 if let Err(qe) = queue_conn.execute(
2856 "UPDATE queue SET status='skipped', error=?1, done_at=datetime('now') WHERE id=?2",
2857 rusqlite::params![reason, queue_id],
2858 ) {
2859 tracing::warn!(target: "enrich", error = %qe, "queue preservation_failed update failed");
2860 }
2861 emit_json(&ItemEvent {
2862 item: &item_key,
2863 status: "preservation_failed",
2864 memory_id: None,
2865 entity_id: None,
2866 entities: None,
2867 rels: None,
2868 chars_before: Some(chars_before),
2869 chars_after: Some(chars_after),
2870 cost_usd: None,
2871 elapsed_ms: Some(item_started.elapsed().as_millis() as u64),
2872 error: Some(reason),
2873 index: current_index,
2874 total,
2875 });
2876 }
2877 Err(e) => {
2878 let err_str = format!("{e}");
2879 if matches!(e, AppError::RateLimited { .. }) {
2880 if crate::retry::is_kill_switch_active() {
2881 tracing::warn!(target: "enrich", "SQLITE_GRAPHRAG_DISABLE_RETRY=1, skipping rate-limit retry");
2882 } else if std::time::Instant::now() >= rate_limit_deadline {
2883 tracing::error!(target: "enrich", total_elapsed_secs = enrich_started.elapsed().as_secs(), "rate-limit retry deadline (1h) exhausted");
2884 } else {
2885 let half = backoff_secs / 2;
2886 let jitter = if half == 0 { 0 } else { fastrand::u64(0..half) };
2887 let actual_wait = half + jitter;
2888 tracing::warn!(target: "enrich", delay_secs = actual_wait, error_kind = "rate_limited", "rate limited, backing off");
2889 if let Err(qe) = queue_conn.execute(
2890 "UPDATE queue SET status='pending' WHERE id=?1",
2891 rusqlite::params![queue_id],
2892 ) {
2893 tracing::warn!(target: "enrich", error = %qe, "queue pending update failed");
2894 }
2895 std::thread::sleep(std::time::Duration::from_secs(actual_wait));
2896 backoff_secs = (backoff_secs * 2).min(900);
2897 continue;
2898 }
2899 }
2900
2901 failed += 1;
2902 let _outcome = match openrouter_diag {
2907 Some(diag) => record_item_failure_typed(
2908 &queue_conn,
2909 queue_id,
2910 attempt_current,
2911 args.max_attempts,
2912 diag.retry_class,
2913 &err_str,
2914 diag.finish_reason.as_deref(),
2915 diag.prompt_tokens,
2916 diag.completion_tokens,
2917 ),
2918 None => record_item_failure(
2919 &queue_conn,
2920 queue_id,
2921 attempt_current,
2922 args.max_attempts,
2923 &e,
2924 ),
2925 };
2926 emit_json(&ItemEvent {
2927 item: &item_key,
2928 status: "failed",
2929 memory_id: None,
2930 entity_id: None,
2931 entities: None,
2932 rels: None,
2933 chars_before: None,
2934 chars_after: None,
2935 cost_usd: None,
2936 elapsed_ms: Some(item_started.elapsed().as_millis() as u64),
2937 error: Some(err_str),
2938 index: current_index,
2939 total,
2940 });
2941 }
2942 }
2943
2944 let _ = item_type; }
2946 } if !args.until_empty {
2949 break;
2950 }
2951 let eligible_remaining: i64 = queue_conn
2952 .query_row(
2953 &format!("SELECT COUNT(*) FROM queue WHERE status='pending' {backoff_clause}"),
2954 [],
2955 |r| r.get(0),
2956 )
2957 .unwrap_or(0);
2958 let progressed = completed > completed_before;
2959 if std::time::Instant::now() >= until_deadline {
2960 tracing::info!(target: "enrich", "until-empty: max-runtime reached, stopping");
2961 break;
2962 }
2963 if !progressed && eligible_remaining == 0 {
2964 tracing::info!(target: "enrich", "until-empty: converged (no eligible items remain)");
2965 break;
2966 }
2967 if eligible_remaining == 0 {
2968 std::thread::sleep(std::time::Duration::from_secs(1));
2970 }
2971 } if crate::shutdown_requested() {
2981 let reset = queue_conn
2982 .execute(
2983 "UPDATE queue SET status='pending', claimed_at=NULL WHERE status='processing'",
2984 [],
2985 )
2986 .unwrap_or(0);
2987 let _ = queue_conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);");
2988 tracing::info!(
2989 target: "enrich",
2990 reset,
2991 "graceful shutdown: WAL checkpointed, processing claims reset"
2992 );
2993 }
2994
2995 let _ = conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);");
2996 let _ = queue_conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);");
2997
2998 let waiting_final: i64 = queue_conn
3002 .query_row(
3003 "SELECT COUNT(*) FROM queue WHERE status='pending' \
3004 AND (operation = ?1 OR operation IS NULL) \
3005 AND next_retry_at IS NOT NULL AND next_retry_at > datetime('now')",
3006 rusqlite::params![op_label],
3007 |r| r.get(0),
3008 )
3009 .unwrap_or(0);
3010 let dead_final: i64 = queue_conn
3011 .query_row(
3012 "SELECT COUNT(*) FROM queue WHERE status='dead' \
3013 AND (operation = ?1 OR operation IS NULL)",
3014 rusqlite::params![op_label],
3015 |r| r.get(0),
3016 )
3017 .unwrap_or(0);
3018
3019 emit_json(&EnrichSummary {
3020 summary: true,
3021 operation: format!("{:?}", args.operation()),
3022 items_total: total,
3023 completed,
3024 failed,
3025 skipped,
3026 cost_usd: cost_total,
3027 elapsed_ms: started.elapsed().as_millis() as u64,
3028 backend_invoked: take_enrich_backend(),
3029 waiting: waiting_final,
3030 dead: dead_final,
3031 });
3032
3033 if failed == 0 {
3034 let dead: i64 = queue_conn
3037 .query_row("SELECT COUNT(*) FROM queue WHERE status='dead'", [], |r| {
3038 r.get(0)
3039 })
3040 .unwrap_or(0);
3041 let skipped_remaining: i64 = queue_conn
3047 .query_row(
3048 "SELECT COUNT(*) FROM queue WHERE status='skipped'",
3049 [],
3050 |r| r.get(0),
3051 )
3052 .unwrap_or(0);
3053 if dead == 0 && skipped_remaining == 0 {
3054 let _ = std::fs::remove_file(&queue_path);
3055 }
3056 }
3057
3058 Ok(())
3059}
3060
3061#[cfg(test)]
3068mod tests {
3069 use super::*;
3070 use rusqlite::ErrorCode;
3071 use std::time::{Duration, Instant};
3072
3073 #[test]
3074 fn bindings_schema_is_valid_json() {
3075 let _: serde_json::Value =
3076 serde_json::from_str(BINDINGS_SCHEMA).expect("BINDINGS_SCHEMA must be valid JSON");
3077 }
3078
3079 #[test]
3080 fn entity_description_schema_is_valid_json() {
3081 let _: serde_json::Value = serde_json::from_str(ENTITY_DESCRIPTION_SCHEMA)
3082 .expect("ENTITY_DESCRIPTION_SCHEMA must be valid JSON");
3083 }
3084
3085 #[test]
3086 fn body_enrich_schema_is_valid_json() {
3087 let _: serde_json::Value = serde_json::from_str(BODY_ENRICH_SCHEMA)
3088 .expect("BODY_ENRICH_SCHEMA must be valid JSON");
3089 }
3090
3091 #[test]
3094 fn enrich_operation_cli_name_pair_ops_are_kebab_case() {
3095 assert_eq!(
3096 enrich_operation_cli_name(&EnrichOperation::EntityConnect),
3097 "entity-connect"
3098 );
3099 assert_eq!(
3100 enrich_operation_cli_name(&EnrichOperation::CrossDomainBridges),
3101 "cross-domain-bridges"
3102 );
3103 assert_eq!(
3104 enrich_operation_cli_name(&EnrichOperation::EntityDescriptions),
3105 "entity-descriptions"
3106 );
3107 }
3108
3109 #[test]
3110 fn is_sqlite_interrupt_detects_operation_interrupted() {
3111 let ffi_err = rusqlite::ffi::Error {
3112 code: ErrorCode::OperationInterrupted,
3113 extended_code: 9,
3114 };
3115 let err = rusqlite::Error::SqliteFailure(ffi_err, Some("interrupted".into()));
3116 assert!(is_sqlite_interrupt(&err));
3117
3118 let busy = rusqlite::ffi::Error {
3119 code: ErrorCode::DatabaseBusy,
3120 extended_code: 5,
3121 };
3122 let busy_err = rusqlite::Error::SqliteFailure(busy, None);
3123 assert!(!is_sqlite_interrupt(&busy_err));
3124 }
3125
3126 #[test]
3127 fn scan_deadline_already_elapsed_returns_timeout() {
3128 use clap::Parser;
3130 let cli = crate::cli::Cli::try_parse_from([
3131 "sqlite-graphrag",
3132 "enrich",
3133 "--operation",
3134 "entity-connect",
3135 "--mode",
3136 "openrouter",
3137 "--openrouter-model",
3138 "test/model",
3139 "--dry-run",
3140 "--limit",
3141 "1",
3142 ])
3143 .expect("parse enrich args");
3144 let Some(crate::cli::Commands::Enrich(args)) = cli.command else {
3145 panic!("expected Commands::Enrich");
3146 };
3147 let conn = Connection::open_in_memory().unwrap();
3148 let past = Instant::now() - Duration::from_secs(1);
3149 let err = scan_operation_with_deadline(&conn, "global", &args, Some(past))
3150 .expect_err("elapsed deadline must Timeout");
3151 match err {
3152 AppError::Timeout { .. } => {}
3153 other => panic!("expected Timeout, got {other:?}"),
3154 }
3155 }
3156
3157 #[test]
3158 fn interrupt_handle_maps_long_query_to_sqlite_interrupt() {
3159 let conn = Connection::open_in_memory().unwrap();
3162 let handle = conn.get_interrupt_handle();
3163 let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
3164 let stop_w = std::sync::Arc::clone(&stop);
3165 let watchdog = std::thread::spawn(move || {
3166 std::thread::sleep(Duration::from_millis(30));
3167 if !stop_w.load(std::sync::atomic::Ordering::Relaxed) {
3168 handle.interrupt();
3169 }
3170 });
3171 let result = conn.query_row(
3172 "WITH RECURSIVE t(x) AS (
3173 SELECT 1
3174 UNION ALL
3175 SELECT x + 1 FROM t WHERE x < 500000000
3176 )
3177 SELECT COUNT(*) FROM t",
3178 [],
3179 |r| r.get::<_, i64>(0),
3180 );
3181 stop.store(true, std::sync::atomic::Ordering::Relaxed);
3182 let _ = watchdog.join();
3183 let err = result.expect_err("recursive CTE must be interrupted");
3184 assert!(
3185 is_sqlite_interrupt(&err),
3186 "expected SQLITE_INTERRUPT, got {err:?}"
3187 );
3188 }
3189}