1use super::args::{resolve_parallelism, IngestArgs, IngestMode};
4use super::persist::{init_storage, persist_staged};
5use super::report::{
6 FileSuccess, IngestDryRunBudget, IngestFileEvent, IngestSummary, StageProgressEvent,
7};
8use super::scan_fs::{collect_files, derive_kebab_name, unique_name, validate_name_prefix};
9use super::stage::{stage_file, StagedFile};
10use super::validate::validate_mode_conditional_flags_ingest;
11use crate::chunking;
12use crate::constants::DERIVED_NAME_MAX_LEN;
13use crate::errors::AppError;
14use crate::output;
15use crate::paths::AppPaths;
16use rayon::iter::{IntoParallelIterator, ParallelIterator};
17use std::collections::BTreeSet;
18use std::path::PathBuf;
19use std::sync::mpsc;
20
21pub fn run(
23 args: IngestArgs,
24 llm_backend: crate::cli::LlmBackendChoice,
25 embedding_backend: crate::cli::EmbeddingBackendChoice,
26) -> Result<(), AppError> {
27 validate_mode_conditional_flags_ingest(&args)?;
30 tracing::debug!(target: "ingest", dir = %args.dir.display(), mode = ?args.mode, "starting ingest");
31 if args.mode == IngestMode::ClaudeCode {
32 return crate::commands::ingest_claude::run_claude_ingest(&args, embedding_backend, llm_backend);
33 }
34 if args.mode == IngestMode::Codex {
35 return crate::commands::ingest_codex::run_codex_ingest(&args);
36 }
37 if args.mode == IngestMode::Opencode {
38 return crate::commands::ingest_opencode::run_opencode_ingest(&args);
39 }
40
41 let started = std::time::Instant::now();
42
43 if !args.dir.exists() {
44 return Err(AppError::Validation(
45 crate::i18n::validation::directory_not_found(&args.dir.display().to_string()),
46 ));
47 }
48 if !args.dir.is_dir() {
49 return Err(AppError::Validation(
50 crate::i18n::validation::not_a_directory(&args.dir.display().to_string()),
51 ));
52 }
53
54 let mut files: Vec<PathBuf> = Vec::with_capacity(128);
55 collect_files(&args.dir, &args.pattern, args.recursive, &mut files)?;
56 files.sort_unstable();
57
58 if files.len() > args.max_files {
59 return Err(AppError::Validation(
60 crate::i18n::validation::max_files_exceeded_matching(files.len(), args.max_files),
61 ));
62 }
63
64 let namespace = crate::namespace::resolve_namespace(args.namespace.as_deref())?;
65 let memory_type_str = args.r#type.as_str().to_string();
66
67 let paths = AppPaths::resolve(args.db.as_deref())?;
68 let mut conn_or_err = match init_storage(&paths) {
69 Ok(c) => Ok(c),
70 Err(e) => Err(format!("{e}")),
71 };
72
73 let mut succeeded: usize = 0;
74 let mut failed: usize = 0;
75 let mut skipped: usize = 0;
76 let total = files.len();
77
78 let mut taken_names: BTreeSet<String> = BTreeSet::new();
81
82 enum SlotMeta {
88 Skip {
89 file_str: String,
90 derived_base: String,
91 name_truncated: bool,
92 original_name: Option<String>,
93 original_filename: Option<String>,
94 reason: String,
95 },
96 Process {
97 file_str: String,
98 derived_name: String,
99 name_truncated: bool,
100 original_name: Option<String>,
101 original_filename: Option<String>,
102 },
103 }
104
105 struct ProcessItem {
106 idx: usize,
107 path: PathBuf,
108 file_str: String,
109 derived_name: String,
110 }
111
112 let files_cap = files.len();
113 let mut slots_meta: Vec<SlotMeta> = Vec::new();
114 slots_meta.try_reserve(files_cap).map_err(|_| {
115 AppError::LimitExceeded(format!(
116 "allocation of {files_cap} slot metadata entries would exceed available memory"
117 ))
118 })?;
119 let mut process_items: Vec<ProcessItem> = Vec::new();
120 process_items.try_reserve(files_cap).map_err(|_| {
121 AppError::LimitExceeded(format!(
122 "allocation of {files_cap} process items would exceed available memory"
123 ))
124 })?;
125 let mut truncations: Vec<(String, String)> = Vec::new();
126 truncations.try_reserve(files_cap).map_err(|_| {
127 AppError::LimitExceeded(format!(
128 "allocation of {files_cap} truncation entries would exceed available memory"
129 ))
130 })?;
131
132 let max_name_length = match args.name_prefix.as_deref() {
135 Some(prefix) => validate_name_prefix(prefix, args.max_name_length)?,
136 None => args.max_name_length,
137 };
138 for path in &files {
139 let file_str = path.to_string_lossy().into_owned();
140 let (derived_base, name_truncated, original_name) =
141 derive_kebab_name(path, max_name_length);
142 let original_basename = path.file_stem().and_then(|s| s.to_str()).unwrap_or("");
143
144 if name_truncated {
145 if let Some(ref orig) = original_name {
146 truncations.push((orig.clone(), derived_base.clone()));
147 }
148 }
149
150 if derived_base.is_empty() {
151 let orig_filename = if !original_basename.is_empty() {
153 Some(original_basename.to_string())
154 } else {
155 None
156 };
157 slots_meta.push(SlotMeta::Skip {
158 file_str,
159 derived_base: String::new(),
160 name_truncated: false,
161 original_name: None,
162 original_filename: orig_filename,
163 reason: "could not derive a non-empty kebab-case name from filename".to_string(),
164 });
165 continue;
166 }
167
168 let derived_base = match args.name_prefix.as_deref() {
172 Some(prefix) => format!("{prefix}{derived_base}"),
173 None => derived_base,
174 };
175
176 match unique_name(&derived_base, &taken_names) {
177 Ok(derived_name) => {
178 taken_names.insert(derived_name.clone());
179 let idx = slots_meta.len();
180 let orig_filename = if original_basename != derived_name {
182 Some(original_basename.to_string())
183 } else {
184 None
185 };
186 process_items.push(ProcessItem {
187 idx,
188 path: path.clone(),
189 file_str: file_str.clone(),
190 derived_name: derived_name.clone(),
191 });
192 slots_meta.push(SlotMeta::Process {
193 file_str,
194 derived_name,
195 name_truncated,
196 original_name,
197 original_filename: orig_filename,
198 });
199 }
200 Err(e) => {
201 let orig_filename = if original_basename != derived_base {
202 Some(original_basename.to_string())
203 } else {
204 None
205 };
206 slots_meta.push(SlotMeta::Skip {
207 file_str,
208 derived_base,
209 name_truncated,
210 original_name,
211 original_filename: orig_filename,
212 reason: e.to_string(),
213 });
214 }
215 }
216 }
217
218 if !truncations.is_empty() {
219 tracing::info!(
220 target: "ingest",
221 count = truncations.len(),
222 max_name_length = max_name_length,
223 max_len = DERIVED_NAME_MAX_LEN,
224 "derived names truncated; pass -vv (debug) for per-file detail"
225 );
226 }
227
228 if args.dry_run {
230 for meta in &slots_meta {
231 match meta {
232 SlotMeta::Skip {
233 file_str,
234 derived_base,
235 name_truncated,
236 original_name,
237 original_filename,
238 reason,
239 } => {
240 output::emit_json_compact(&IngestFileEvent {
241 file: file_str,
242 name: derived_base,
243 status: "skip",
244 truncated: *name_truncated,
245 original_name: original_name.clone(),
246 original_filename: original_filename.as_deref(),
247 error: Some(reason.clone()),
248 memory_id: None,
249 action: None,
250 body_length: 0,
251 backend_invoked: None,
252 })?;
253 }
254 SlotMeta::Process {
255 file_str,
256 derived_name,
257 name_truncated,
258 original_name,
259 original_filename,
260 } => {
261 output::emit_json_compact(&IngestFileEvent {
262 file: file_str,
263 name: derived_name,
264 status: "preview",
265 truncated: *name_truncated,
266 original_name: original_name.clone(),
267 original_filename: original_filename.as_deref(),
268 error: None,
269 memory_id: None,
270 action: None,
271 body_length: 0,
272 backend_invoked: None,
273 })?;
274
275 match std::fs::read_to_string(file_str) {
279 Ok(body) => {
280 let budget = chunking::assess_body_budget(&body);
281 output::emit_json_compact(&IngestDryRunBudget {
282 budget: true,
283 file: file_str,
284 name: derived_name,
285 bytes: budget.bytes,
286 chunk_count: budget.chunk_count,
287 token_count: budget.approx_tokens,
288 partition_count: budget.partition_count,
289 exceeds_limits: budget.exceeds_limits,
290 })?;
291 }
292 Err(e) => {
293 tracing::warn!(
294 target: "ingest",
295 file = %file_str,
296 "dry-run: could not read file for budget assessment: {e}"
297 );
298 }
299 }
300 }
301 }
302 }
303 output::emit_json_compact(&IngestSummary {
304 summary: true,
305 dir: args.dir.to_string_lossy().into_owned(),
306 pattern: args.pattern.clone(),
307 recursive: args.recursive,
308 files_total: total,
309 files_succeeded: 0,
310 files_failed: 0,
311 files_skipped: 0,
312 elapsed_ms: started.elapsed().as_millis() as u64,
313 })?;
314 return Ok(());
315 }
316
317 if args.low_memory {
319 if let Some(n) = args.ingest_parallelism {
320 if n > 1 {
321 return Err(AppError::Validation(
322 "--ingest-parallelism N>1 conflicts with --low-memory; use one or the other"
323 .to_string(),
324 ));
325 }
326 }
327 }
328
329 let parallelism = resolve_parallelism(args.low_memory, args.ingest_parallelism);
332
333 let pool = rayon::ThreadPoolBuilder::new()
334 .num_threads(parallelism)
335 .build()
336 .map_err(|e| AppError::Internal(anyhow::anyhow!("rayon pool: {e}")))?;
337
338 if args.enable_ner && args.skip_extraction {
339 return Err(AppError::Validation(
340 crate::i18n::validation::enable_ner_skip_extraction_exclusive(),
341 ));
342 }
343 if args.skip_extraction && !args.enable_ner {
344 tracing::warn!(
351 "--skip-extraction is deprecated since v1.0.45 and has no effect (NER is disabled by default); remove this flag to silence the warning"
352 );
353 }
354 let enable_ner = args.enable_ner;
355 let auto_describe = args.auto_describe && !args.no_auto_describe;
356 let max_rss_mb = args.max_rss_mb;
357 let llm_parallelism = args.llm_parallelism as usize;
358
359 let total_to_process = process_items.len();
360 tracing::info!(
361 target: "ingest",
362 phase = "pipeline_start",
363 files = total_to_process,
364 ingest_parallelism = parallelism,
365 "incremental pipeline starting: Phase A (rayon) → channel → Phase B (main thread)",
366 );
367
368 let channel_bound = (parallelism * 2).max(1);
372 let (tx, rx) = mpsc::sync_channel::<(usize, Result<Vec<StagedFile>, AppError>)>(channel_bound);
373
374 let paths_owned = paths.clone();
379 let llm_backend_owned = llm_backend;
380 let embedding_backend_owned = embedding_backend;
381 let producer_handle = std::thread::spawn(move || {
382 pool.install(|| {
383 process_items.into_par_iter().for_each(|item| {
384 if crate::shutdown_requested() {
385 return;
386 }
387 let t0 = std::time::Instant::now();
388 let result = stage_file(
389 item.idx,
390 &item.path,
391 &item.derived_name,
392 &paths_owned,
393 enable_ner,
394 max_rss_mb,
395 llm_parallelism,
396 llm_backend_owned,
397 embedding_backend_owned,
398 auto_describe,
399 );
400 let elapsed_ms = t0.elapsed().as_millis() as u64;
401
402 let (n_entities, n_relationships) = match &result {
405 Ok(parts) => (
406 parts.iter().map(|sf| sf.entities.len()).sum::<usize>(),
407 parts.iter().map(|sf| sf.relationships.len()).sum::<usize>(),
408 ),
409 Err(_) => (0, 0),
410 };
411 let progress = StageProgressEvent {
412 schema_version: 1,
413 event: "file_extracted",
414 path: &item.file_str,
415 ms: elapsed_ms,
416 entities: n_entities,
417 relationships: n_relationships,
418 };
419 if let Ok(line) = serde_json::to_string(&progress) {
420 tracing::info!(target: "ingest_progress", "{}", line);
421 }
422
423 let _ = tx.send((item.idx, result));
427 });
428 drop(tx);
430 });
431 });
432
433 let fail_fast = args.fail_fast;
445
446 for meta in &slots_meta {
448 if let SlotMeta::Skip {
449 file_str,
450 derived_base,
451 name_truncated,
452 original_name,
453 original_filename,
454 reason,
455 } = meta
456 {
457 output::emit_json_compact(&IngestFileEvent {
458 file: file_str,
459 name: derived_base,
460 status: "skipped",
461 truncated: *name_truncated,
462 original_name: original_name.clone(),
463 original_filename: original_filename.as_deref(),
464 error: Some(reason.clone()),
465 memory_id: None,
466 action: None,
467 body_length: 0,
468 backend_invoked: None,
469 })?;
470 skipped += 1;
471 }
472 }
473
474 let meta_index: std::collections::HashMap<usize, &SlotMeta> = slots_meta
477 .iter()
478 .enumerate()
479 .filter(|(_, m)| matches!(m, SlotMeta::Process { .. }))
480 .collect();
481
482 tracing::info!(
483 target: "ingest",
484 phase = "persist_start",
485 files = total_to_process,
486 "phase B starting: persisting files incrementally as Phase A completes each one",
487 );
488
489 for (idx, stage_result) in rx {
493 if crate::shutdown_requested() {
494 tracing::info!(target: "ingest", "shutdown requested, stopping persistence loop");
495 break;
496 }
497 let meta = meta_index.get(&idx).ok_or_else(|| {
498 AppError::Internal(anyhow::anyhow!(
499 "channel idx {idx} has no corresponding Process slot"
500 ))
501 })?;
502 let (file_str, derived_name, name_truncated, original_name, original_filename) = match meta
503 {
504 SlotMeta::Process {
505 file_str,
506 derived_name,
507 name_truncated,
508 original_name,
509 original_filename,
510 } => (
511 file_str,
512 derived_name,
513 name_truncated,
514 original_name,
515 original_filename,
516 ),
517 SlotMeta::Skip { .. } => unreachable!("channel only carries Process results"),
518 };
519
520 let conn = match conn_or_err.as_mut() {
522 Ok(c) => c,
523 Err(err_msg) => {
524 let err_clone = err_msg.clone();
525 output::emit_json_compact(&IngestFileEvent {
526 file: file_str,
527 name: derived_name,
528 status: "failed",
529 truncated: *name_truncated,
530 original_name: original_name.clone(),
531 original_filename: original_filename.as_deref(),
532 error: Some(err_clone.clone()),
533 memory_id: None,
534 action: None,
535 body_length: 0,
536 backend_invoked: None,
537 })?;
538 failed += 1;
539 if fail_fast {
540 output::emit_json_compact(&IngestSummary {
541 summary: true,
542 dir: args.dir.display().to_string(),
543 pattern: args.pattern.clone(),
544 recursive: args.recursive,
545 files_total: total,
546 files_succeeded: succeeded,
547 files_failed: failed,
548 files_skipped: skipped,
549 elapsed_ms: started.elapsed().as_millis() as u64,
550 })?;
551 return Err(AppError::Validation(
552 crate::i18n::validation::ingest_aborted_on_first_failure(&err_clone),
553 ));
554 }
555 continue;
556 }
557 };
558
559 match stage_result {
560 Ok(parts) => {
561 for staged in parts {
564 let part_name = staged.name.clone();
565 match persist_staged(
566 conn,
567 &namespace,
568 &memory_type_str,
569 staged,
570 args.force_merge,
571 ) {
572 Ok(FileSuccess {
573 memory_id,
574 action,
575 body_length,
576 backend_invoked: file_backend_invoked,
577 }) => {
578 output::emit_json_compact(&IngestFileEvent {
579 file: file_str,
580 name: &part_name,
581 status: "indexed",
582 truncated: *name_truncated,
583 original_name: original_name.clone(),
584 original_filename: original_filename.as_deref(),
585 error: None,
586 memory_id: Some(memory_id),
587 action: Some(action),
588 body_length,
589 backend_invoked: file_backend_invoked,
590 })?;
591 succeeded += 1;
592 }
593 Err(ref e) if matches!(e, AppError::Duplicate(_)) => {
594 output::emit_json_compact(&IngestFileEvent {
595 file: file_str,
596 name: &part_name,
597 status: "skipped",
598 truncated: *name_truncated,
599 original_name: original_name.clone(),
600 original_filename: original_filename.as_deref(),
601 error: Some(format!("{e}")),
602 memory_id: None,
603 action: Some("duplicate".to_string()),
604 body_length: 0,
605 backend_invoked: None,
606 })?;
607 skipped += 1;
608 }
609 Err(e) => {
610 let err_msg = format!("{e}");
611 output::emit_json_compact(&IngestFileEvent {
612 file: file_str,
613 name: &part_name,
614 status: "failed",
615 truncated: *name_truncated,
616 original_name: original_name.clone(),
617 original_filename: original_filename.as_deref(),
618 error: Some(err_msg.clone()),
619 memory_id: None,
620 action: None,
621 body_length: 0,
622 backend_invoked: None,
623 })?;
624 failed += 1;
625 if fail_fast {
626 output::emit_json_compact(&IngestSummary {
627 summary: true,
628 dir: args.dir.display().to_string(),
629 pattern: args.pattern.clone(),
630 recursive: args.recursive,
631 files_total: total,
632 files_succeeded: succeeded,
633 files_failed: failed,
634 files_skipped: skipped,
635 elapsed_ms: started.elapsed().as_millis() as u64,
636 })?;
637 return Err(AppError::Validation(
638 crate::i18n::validation::ingest_aborted_on_first_failure(
639 &err_msg,
640 ),
641 ));
642 }
643 }
644 }
645 }
646 }
647 Err(e) => {
648 let err_msg = format!("{e}");
649 output::emit_json_compact(&IngestFileEvent {
650 file: file_str,
651 name: derived_name,
652 status: "failed",
653 truncated: *name_truncated,
654 original_name: original_name.clone(),
655 original_filename: original_filename.as_deref(),
656 error: Some(err_msg.clone()),
657 memory_id: None,
658 action: None,
659 body_length: 0,
660 backend_invoked: None,
661 })?;
662 failed += 1;
663 if fail_fast {
664 output::emit_json_compact(&IngestSummary {
665 summary: true,
666 dir: args.dir.display().to_string(),
667 pattern: args.pattern.clone(),
668 recursive: args.recursive,
669 files_total: total,
670 files_succeeded: succeeded,
671 files_failed: failed,
672 files_skipped: skipped,
673 elapsed_ms: started.elapsed().as_millis() as u64,
674 })?;
675 return Err(AppError::Validation(
676 crate::i18n::validation::ingest_aborted_on_first_failure(&err_msg),
677 ));
678 }
679 }
680 }
681 }
682
683 producer_handle
685 .join()
686 .map_err(|_| AppError::Internal(anyhow::anyhow!("ingest producer thread panicked")))?;
687
688 if let Ok(ref conn) = conn_or_err {
689 if succeeded > 0 {
690 let _ = conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);");
691 }
692 }
693
694 output::emit_json_compact(&IngestSummary {
695 summary: true,
696 dir: args.dir.display().to_string(),
697 pattern: args.pattern.clone(),
698 recursive: args.recursive,
699 files_total: total,
700 files_succeeded: succeeded,
701 files_failed: failed,
702 files_skipped: skipped,
703 elapsed_ms: started.elapsed().as_millis() as u64,
704 })?;
705
706 if args.enrich_after && succeeded > 0 {
707 output::emit_json_compact(&serde_json::json!({
708 "event": "enrich_phase_started",
709 "operation": "memory-bindings"
710 }))?;
711 let enrich_args = crate::commands::enrich::EnrichArgs {
712 operation: Some(crate::commands::enrich::EnrichOperation::MemoryBindings),
713 mode: Some(crate::commands::enrich::EnrichMode::ClaudeCode),
714 limit: None,
715 target: crate::commands::enrich::ReEmbedTarget::Memories,
716 dry_run: false,
717 namespace: args.namespace.clone(),
718 claude_binary: args.claude_binary.clone(),
719 claude_model: args.claude_model.clone(),
720 claude_timeout: args.claude_timeout,
721 codex_binary: args.codex_binary.clone(),
722 codex_model: args.codex_model.clone(),
723 codex_timeout: args.codex_timeout,
724 opencode_binary: args.opencode_binary.clone(),
725 opencode_model: args.opencode_model.clone(),
726 opencode_timeout: args.opencode_timeout,
727 openrouter_model: None,
728 openrouter_api_key: None,
729 openrouter_timeout: 300,
730 openrouter_base_url: None,
731 db: args.db.clone(),
732 json: false,
733 resume: false,
734 retry_failed: false,
735 reset_stale_claims: false,
736 stale_claim_secs: 1800,
737 max_cost_usd: args.max_cost_usd,
738 llm_parallelism: args.llm_parallelism as u32,
739 wait_job_singleton: args.wait_job_singleton,
740 force_job_singleton: args.force_job_singleton,
741 names: Vec::new(),
742 names_file: None,
743 preflight_check: false,
744 fallback_mode: None,
745 rate_limit_buffer: 300,
746 max_load_check: true,
747 circuit_breaker_threshold: 5,
748 preserve_threshold: 0.7,
749 entity_description_grounding_threshold: 0.12,
750 force_redescribe: false,
751 quality_sample: None,
752 entity_names: Vec::new(),
753 memory_names: Vec::new(),
754 anchor_memory: None,
755 entity_description_domain: "auto".to_string(),
756 yield_every_n_items: None,
757 ops_gate: false,
758 codex_model_validate: true,
759 codex_model_fallback: None,
760 min_output_chars: 500,
761 max_output_chars: 2000,
762 preserve_check: true,
763 prompt_template: None,
764 until_empty: false,
765 max_runtime: None,
766 max_attempts: 5,
767 status: false,
768 rest_concurrency: None,
769 list_dead: false,
772 requeue_dead: false,
773 list_skipped: false,
774 requeue_skipped: false,
775 prune_dead_orphans: false,
776 prune_dead_entity_orphans: false,
777 ignore_backoff: false,
778 body_extract_graph_only: false,
779 print_schema: false,
780 };
781 match crate::commands::enrich::run(&enrich_args, llm_backend, embedding_backend) {
782 Ok(()) => {
783 output::emit_json_compact(&serde_json::json!({
784 "event": "enrich_phase_completed"
785 }))?;
786 }
787 Err(e) => {
788 tracing::warn!(error = %e, "enrich --operation memory-bindings failed after ingest");
789 output::emit_json_compact(&serde_json::json!({
790 "event": "enrich_phase_failed",
791 "error": e.to_string()
792 }))?;
793 }
794 }
795 }
796
797 Ok(())
798}