1use super::args::{resolve_parallelism, IngestArgs, IngestMode};
4use super::persist::{init_storage, persist_staged};
5use super::report::{
6 FileSuccess, IngestDryRunBudget, IngestFileEvent, IngestSummary, StageProgressEvent,
7};
8use super::scan_fs::{collect_files, derive_kebab_name, unique_name, validate_name_prefix};
9use super::stage::{stage_file, StagedFile};
10use super::validate::validate_mode_conditional_flags_ingest;
11use crate::chunking;
12use crate::constants::DERIVED_NAME_MAX_LEN;
13use crate::errors::AppError;
14use crate::output;
15use crate::paths::AppPaths;
16use rayon::iter::{IntoParallelIterator, ParallelIterator};
17use std::collections::BTreeSet;
18use std::path::PathBuf;
19use std::sync::mpsc;
20
21pub fn run(
23 args: IngestArgs,
24 llm_backend: crate::cli::LlmBackendChoice,
25 embedding_backend: crate::cli::EmbeddingBackendChoice,
26) -> Result<(), AppError> {
27 validate_mode_conditional_flags_ingest(&args)?;
30 tracing::debug!(target: "ingest", dir = %args.dir.display(), mode = ?args.mode, "starting ingest");
31 if args.mode == IngestMode::ClaudeCode {
32 return crate::commands::ingest_claude::run_claude_ingest(
33 &args,
34 embedding_backend,
35 llm_backend,
36 );
37 }
38 if args.mode == IngestMode::Codex {
39 return crate::commands::ingest_codex::run_codex_ingest(&args);
40 }
41 if args.mode == IngestMode::Opencode {
42 return crate::commands::ingest_opencode::run_opencode_ingest(&args);
43 }
44
45 let started = std::time::Instant::now();
46
47 if !args.dir.exists() {
48 return Err(AppError::Validation(
49 crate::i18n::validation::directory_not_found(&args.dir.display().to_string()),
50 ));
51 }
52 if !args.dir.is_dir() {
53 return Err(AppError::Validation(
54 crate::i18n::validation::not_a_directory(&args.dir.display().to_string()),
55 ));
56 }
57
58 let mut files: Vec<PathBuf> = Vec::with_capacity(128);
59 collect_files(&args.dir, &args.pattern, args.recursive, &mut files)?;
60 files.sort_unstable();
61
62 if files.len() > args.max_files {
63 return Err(AppError::Validation(
64 crate::i18n::validation::max_files_exceeded_matching(files.len(), args.max_files),
65 ));
66 }
67
68 let namespace = crate::namespace::resolve_namespace(args.namespace.as_deref())?;
69 let memory_type_str = args.r#type.as_str().to_string();
70
71 let paths = AppPaths::resolve(args.db.as_deref())?;
72 let mut conn_or_err = match init_storage(&paths) {
73 Ok(c) => Ok(c),
74 Err(e) => Err(format!("{e}")),
75 };
76
77 let mut succeeded: usize = 0;
78 let mut failed: usize = 0;
79 let mut skipped: usize = 0;
80 let total = files.len();
81
82 let mut taken_names: BTreeSet<String> = BTreeSet::new();
85
86 enum SlotMeta {
92 Skip {
93 file_str: String,
94 derived_base: String,
95 name_truncated: bool,
96 original_name: Option<String>,
97 original_filename: Option<String>,
98 reason: String,
99 },
100 Process {
101 file_str: String,
102 derived_name: String,
103 name_truncated: bool,
104 original_name: Option<String>,
105 original_filename: Option<String>,
106 },
107 }
108
109 struct ProcessItem {
110 idx: usize,
111 path: PathBuf,
112 file_str: String,
113 derived_name: String,
114 }
115
116 let files_cap = files.len();
117 let mut slots_meta: Vec<SlotMeta> = Vec::new();
118 slots_meta.try_reserve(files_cap).map_err(|_| {
119 AppError::LimitExceeded(format!(
120 "allocation of {files_cap} slot metadata entries would exceed available memory"
121 ))
122 })?;
123 let mut process_items: Vec<ProcessItem> = Vec::new();
124 process_items.try_reserve(files_cap).map_err(|_| {
125 AppError::LimitExceeded(format!(
126 "allocation of {files_cap} process items would exceed available memory"
127 ))
128 })?;
129 let mut truncations: Vec<(String, String)> = Vec::new();
130 truncations.try_reserve(files_cap).map_err(|_| {
131 AppError::LimitExceeded(format!(
132 "allocation of {files_cap} truncation entries would exceed available memory"
133 ))
134 })?;
135
136 let max_name_length = match args.name_prefix.as_deref() {
139 Some(prefix) => validate_name_prefix(prefix, args.max_name_length)?,
140 None => args.max_name_length,
141 };
142 for path in &files {
143 let file_str = path.to_string_lossy().into_owned();
144 let (derived_base, name_truncated, original_name) =
145 derive_kebab_name(path, max_name_length);
146 let original_basename = path.file_stem().and_then(|s| s.to_str()).unwrap_or("");
147
148 if name_truncated {
149 if let Some(ref orig) = original_name {
150 truncations.push((orig.clone(), derived_base.clone()));
151 }
152 }
153
154 if derived_base.is_empty() {
155 let orig_filename = if !original_basename.is_empty() {
157 Some(original_basename.to_string())
158 } else {
159 None
160 };
161 slots_meta.push(SlotMeta::Skip {
162 file_str,
163 derived_base: String::new(),
164 name_truncated: false,
165 original_name: None,
166 original_filename: orig_filename,
167 reason: "could not derive a non-empty kebab-case name from filename".to_string(),
168 });
169 continue;
170 }
171
172 let derived_base = match args.name_prefix.as_deref() {
176 Some(prefix) => format!("{prefix}{derived_base}"),
177 None => derived_base,
178 };
179
180 match unique_name(&derived_base, &taken_names) {
181 Ok(derived_name) => {
182 taken_names.insert(derived_name.clone());
183 let idx = slots_meta.len();
184 let orig_filename = if original_basename != derived_name {
186 Some(original_basename.to_string())
187 } else {
188 None
189 };
190 process_items.push(ProcessItem {
191 idx,
192 path: path.clone(),
193 file_str: file_str.clone(),
194 derived_name: derived_name.clone(),
195 });
196 slots_meta.push(SlotMeta::Process {
197 file_str,
198 derived_name,
199 name_truncated,
200 original_name,
201 original_filename: orig_filename,
202 });
203 }
204 Err(e) => {
205 let orig_filename = if original_basename != derived_base {
206 Some(original_basename.to_string())
207 } else {
208 None
209 };
210 slots_meta.push(SlotMeta::Skip {
211 file_str,
212 derived_base,
213 name_truncated,
214 original_name,
215 original_filename: orig_filename,
216 reason: e.to_string(),
217 });
218 }
219 }
220 }
221
222 if !truncations.is_empty() {
223 tracing::info!(
224 target: "ingest",
225 count = truncations.len(),
226 max_name_length = max_name_length,
227 max_len = DERIVED_NAME_MAX_LEN,
228 "derived names truncated; pass -vv (debug) for per-file detail"
229 );
230 }
231
232 if args.dry_run {
234 for meta in &slots_meta {
235 match meta {
236 SlotMeta::Skip {
237 file_str,
238 derived_base,
239 name_truncated,
240 original_name,
241 original_filename,
242 reason,
243 } => {
244 output::emit_json_compact(&IngestFileEvent {
245 file: file_str,
246 name: derived_base,
247 status: "skip",
248 truncated: *name_truncated,
249 original_name: original_name.clone(),
250 original_filename: original_filename.as_deref(),
251 error: Some(reason.clone()),
252 memory_id: None,
253 action: None,
254 body_length: 0,
255 backend_invoked: None,
256 })?;
257 }
258 SlotMeta::Process {
259 file_str,
260 derived_name,
261 name_truncated,
262 original_name,
263 original_filename,
264 } => {
265 output::emit_json_compact(&IngestFileEvent {
266 file: file_str,
267 name: derived_name,
268 status: "preview",
269 truncated: *name_truncated,
270 original_name: original_name.clone(),
271 original_filename: original_filename.as_deref(),
272 error: None,
273 memory_id: None,
274 action: None,
275 body_length: 0,
276 backend_invoked: None,
277 })?;
278
279 match std::fs::read_to_string(file_str) {
283 Ok(body) => {
284 let budget = chunking::assess_body_budget(&body);
285 output::emit_json_compact(&IngestDryRunBudget {
286 budget: true,
287 file: file_str,
288 name: derived_name,
289 bytes: budget.bytes,
290 chunk_count: budget.chunk_count,
291 token_count: budget.approx_tokens,
292 partition_count: budget.partition_count,
293 exceeds_limits: budget.exceeds_limits,
294 })?;
295 }
296 Err(e) => {
297 tracing::warn!(
298 target: "ingest",
299 file = %file_str,
300 "dry-run: could not read file for budget assessment: {e}"
301 );
302 }
303 }
304 }
305 }
306 }
307 output::emit_json_compact(&IngestSummary {
308 summary: true,
309 dir: args.dir.to_string_lossy().into_owned(),
310 pattern: args.pattern.clone(),
311 recursive: args.recursive,
312 files_total: total,
313 files_succeeded: 0,
314 files_failed: 0,
315 files_skipped: 0,
316 elapsed_ms: started.elapsed().as_millis() as u64,
317 })?;
318 return Ok(());
319 }
320
321 if args.low_memory {
323 if let Some(n) = args.ingest_parallelism {
324 if n > 1 {
325 return Err(AppError::Validation(
326 "--ingest-parallelism N>1 conflicts with --low-memory; use one or the other"
327 .to_string(),
328 ));
329 }
330 }
331 }
332
333 let parallelism = resolve_parallelism(args.low_memory, args.ingest_parallelism);
336
337 let pool = rayon::ThreadPoolBuilder::new()
338 .num_threads(parallelism)
339 .build()
340 .map_err(|e| AppError::Internal(anyhow::anyhow!("rayon pool: {e}")))?;
341
342 if args.enable_ner && args.skip_extraction {
343 return Err(AppError::Validation(
344 crate::i18n::validation::enable_ner_skip_extraction_exclusive(),
345 ));
346 }
347 if args.skip_extraction && !args.enable_ner {
348 tracing::warn!(
355 "--skip-extraction is deprecated since v1.0.45 and has no effect (NER is disabled by default); remove this flag to silence the warning"
356 );
357 }
358 let enable_ner = args.enable_ner;
359 let auto_describe = args.auto_describe && !args.no_auto_describe;
360 let max_rss_mb = args.max_rss_mb;
361 let llm_parallelism = args.llm_parallelism as usize;
362
363 let total_to_process = process_items.len();
364 tracing::info!(
365 target: "ingest",
366 phase = "pipeline_start",
367 files = total_to_process,
368 ingest_parallelism = parallelism,
369 "incremental pipeline starting: Phase A (rayon) → channel → Phase B (main thread)",
370 );
371
372 let channel_bound = (parallelism * 2).max(1);
376 let (tx, rx) = mpsc::sync_channel::<(usize, Result<Vec<StagedFile>, AppError>)>(channel_bound);
377
378 let paths_owned = paths.clone();
383 let llm_backend_owned = llm_backend;
384 let embedding_backend_owned = embedding_backend;
385 let producer_handle = std::thread::spawn(move || {
386 pool.install(|| {
387 process_items.into_par_iter().for_each(|item| {
388 if crate::shutdown_requested() {
389 return;
390 }
391 let t0 = std::time::Instant::now();
392 let result = stage_file(
393 item.idx,
394 &item.path,
395 &item.derived_name,
396 &paths_owned,
397 enable_ner,
398 max_rss_mb,
399 llm_parallelism,
400 llm_backend_owned,
401 embedding_backend_owned,
402 auto_describe,
403 );
404 let elapsed_ms = t0.elapsed().as_millis() as u64;
405
406 let (n_entities, n_relationships) = match &result {
409 Ok(parts) => (
410 parts.iter().map(|sf| sf.entities.len()).sum::<usize>(),
411 parts.iter().map(|sf| sf.relationships.len()).sum::<usize>(),
412 ),
413 Err(_) => (0, 0),
414 };
415 let progress = StageProgressEvent {
416 schema_version: 1,
417 event: "file_extracted",
418 path: &item.file_str,
419 ms: elapsed_ms,
420 entities: n_entities,
421 relationships: n_relationships,
422 };
423 if let Ok(line) = serde_json::to_string(&progress) {
424 tracing::info!(target: "ingest_progress", "{}", line);
425 }
426
427 let _ = tx.send((item.idx, result));
431 });
432 drop(tx);
434 });
435 });
436
437 let fail_fast = args.fail_fast;
449
450 for meta in &slots_meta {
452 if let SlotMeta::Skip {
453 file_str,
454 derived_base,
455 name_truncated,
456 original_name,
457 original_filename,
458 reason,
459 } = meta
460 {
461 output::emit_json_compact(&IngestFileEvent {
462 file: file_str,
463 name: derived_base,
464 status: "skipped",
465 truncated: *name_truncated,
466 original_name: original_name.clone(),
467 original_filename: original_filename.as_deref(),
468 error: Some(reason.clone()),
469 memory_id: None,
470 action: None,
471 body_length: 0,
472 backend_invoked: None,
473 })?;
474 skipped += 1;
475 }
476 }
477
478 let meta_index: std::collections::HashMap<usize, &SlotMeta> = slots_meta
481 .iter()
482 .enumerate()
483 .filter(|(_, m)| matches!(m, SlotMeta::Process { .. }))
484 .collect();
485
486 tracing::info!(
487 target: "ingest",
488 phase = "persist_start",
489 files = total_to_process,
490 "phase B starting: persisting files incrementally as Phase A completes each one",
491 );
492
493 for (idx, stage_result) in rx {
497 if crate::shutdown_requested() {
498 tracing::info!(target: "ingest", "shutdown requested, stopping persistence loop");
499 break;
500 }
501 let meta = meta_index.get(&idx).ok_or_else(|| {
502 AppError::Internal(anyhow::anyhow!(
503 "channel idx {idx} has no corresponding Process slot"
504 ))
505 })?;
506 let (file_str, derived_name, name_truncated, original_name, original_filename) = match meta
507 {
508 SlotMeta::Process {
509 file_str,
510 derived_name,
511 name_truncated,
512 original_name,
513 original_filename,
514 } => (
515 file_str,
516 derived_name,
517 name_truncated,
518 original_name,
519 original_filename,
520 ),
521 SlotMeta::Skip { .. } => unreachable!("channel only carries Process results"),
522 };
523
524 let conn = match conn_or_err.as_mut() {
526 Ok(c) => c,
527 Err(err_msg) => {
528 let err_clone = err_msg.clone();
529 output::emit_json_compact(&IngestFileEvent {
530 file: file_str,
531 name: derived_name,
532 status: "failed",
533 truncated: *name_truncated,
534 original_name: original_name.clone(),
535 original_filename: original_filename.as_deref(),
536 error: Some(err_clone.clone()),
537 memory_id: None,
538 action: None,
539 body_length: 0,
540 backend_invoked: None,
541 })?;
542 failed += 1;
543 if fail_fast {
544 output::emit_json_compact(&IngestSummary {
545 summary: true,
546 dir: args.dir.display().to_string(),
547 pattern: args.pattern.clone(),
548 recursive: args.recursive,
549 files_total: total,
550 files_succeeded: succeeded,
551 files_failed: failed,
552 files_skipped: skipped,
553 elapsed_ms: started.elapsed().as_millis() as u64,
554 })?;
555 return Err(AppError::Validation(
556 crate::i18n::validation::ingest_aborted_on_first_failure(&err_clone),
557 ));
558 }
559 continue;
560 }
561 };
562
563 match stage_result {
564 Ok(parts) => {
565 for staged in parts {
568 let part_name = staged.name.clone();
569 match persist_staged(
570 conn,
571 &namespace,
572 &memory_type_str,
573 staged,
574 args.force_merge,
575 ) {
576 Ok(FileSuccess {
577 memory_id,
578 action,
579 body_length,
580 backend_invoked: file_backend_invoked,
581 }) => {
582 output::emit_json_compact(&IngestFileEvent {
583 file: file_str,
584 name: &part_name,
585 status: "indexed",
586 truncated: *name_truncated,
587 original_name: original_name.clone(),
588 original_filename: original_filename.as_deref(),
589 error: None,
590 memory_id: Some(memory_id),
591 action: Some(action),
592 body_length,
593 backend_invoked: file_backend_invoked,
594 })?;
595 succeeded += 1;
596 }
597 Err(ref e) if matches!(e, AppError::Duplicate(_)) => {
598 output::emit_json_compact(&IngestFileEvent {
599 file: file_str,
600 name: &part_name,
601 status: "skipped",
602 truncated: *name_truncated,
603 original_name: original_name.clone(),
604 original_filename: original_filename.as_deref(),
605 error: Some(format!("{e}")),
606 memory_id: None,
607 action: Some("duplicate".to_string()),
608 body_length: 0,
609 backend_invoked: None,
610 })?;
611 skipped += 1;
612 }
613 Err(e) => {
614 let err_msg = format!("{e}");
615 output::emit_json_compact(&IngestFileEvent {
616 file: file_str,
617 name: &part_name,
618 status: "failed",
619 truncated: *name_truncated,
620 original_name: original_name.clone(),
621 original_filename: original_filename.as_deref(),
622 error: Some(err_msg.clone()),
623 memory_id: None,
624 action: None,
625 body_length: 0,
626 backend_invoked: None,
627 })?;
628 failed += 1;
629 if fail_fast {
630 output::emit_json_compact(&IngestSummary {
631 summary: true,
632 dir: args.dir.display().to_string(),
633 pattern: args.pattern.clone(),
634 recursive: args.recursive,
635 files_total: total,
636 files_succeeded: succeeded,
637 files_failed: failed,
638 files_skipped: skipped,
639 elapsed_ms: started.elapsed().as_millis() as u64,
640 })?;
641 return Err(AppError::Validation(
642 crate::i18n::validation::ingest_aborted_on_first_failure(
643 &err_msg,
644 ),
645 ));
646 }
647 }
648 }
649 }
650 }
651 Err(e) => {
652 let err_msg = format!("{e}");
653 output::emit_json_compact(&IngestFileEvent {
654 file: file_str,
655 name: derived_name,
656 status: "failed",
657 truncated: *name_truncated,
658 original_name: original_name.clone(),
659 original_filename: original_filename.as_deref(),
660 error: Some(err_msg.clone()),
661 memory_id: None,
662 action: None,
663 body_length: 0,
664 backend_invoked: None,
665 })?;
666 failed += 1;
667 if fail_fast {
668 output::emit_json_compact(&IngestSummary {
669 summary: true,
670 dir: args.dir.display().to_string(),
671 pattern: args.pattern.clone(),
672 recursive: args.recursive,
673 files_total: total,
674 files_succeeded: succeeded,
675 files_failed: failed,
676 files_skipped: skipped,
677 elapsed_ms: started.elapsed().as_millis() as u64,
678 })?;
679 return Err(AppError::Validation(
680 crate::i18n::validation::ingest_aborted_on_first_failure(&err_msg),
681 ));
682 }
683 }
684 }
685 }
686
687 producer_handle
689 .join()
690 .map_err(|_| AppError::Internal(anyhow::anyhow!("ingest producer thread panicked")))?;
691
692 if let Ok(ref conn) = conn_or_err {
693 if succeeded > 0 {
694 let _ = conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);");
695 }
696 }
697
698 output::emit_json_compact(&IngestSummary {
699 summary: true,
700 dir: args.dir.display().to_string(),
701 pattern: args.pattern.clone(),
702 recursive: args.recursive,
703 files_total: total,
704 files_succeeded: succeeded,
705 files_failed: failed,
706 files_skipped: skipped,
707 elapsed_ms: started.elapsed().as_millis() as u64,
708 })?;
709
710 if args.enrich_after && succeeded > 0 {
711 output::emit_json_compact(&serde_json::json!({
712 "event": "enrich_phase_started",
713 "operation": "memory-bindings"
714 }))?;
715 let enrich_args = crate::commands::enrich::EnrichArgs {
716 operation: Some(crate::commands::enrich::EnrichOperation::MemoryBindings),
717 mode: Some(crate::commands::enrich::EnrichMode::ClaudeCode),
718 limit: None,
719 target: crate::commands::enrich::ReEmbedTarget::Memories,
720 dry_run: false,
721 namespace: args.namespace.clone(),
722 claude_binary: args.claude_binary.clone(),
723 claude_model: args.claude_model.clone(),
724 claude_timeout: args.claude_timeout,
725 codex_binary: args.codex_binary.clone(),
726 codex_model: args.codex_model.clone(),
727 codex_timeout: args.codex_timeout,
728 opencode_binary: args.opencode_binary.clone(),
729 opencode_model: args.opencode_model.clone(),
730 opencode_timeout: args.opencode_timeout,
731 openrouter_model: None,
732 openrouter_api_key: None,
733 openrouter_timeout: 300,
734 openrouter_base_url: None,
735 db: args.db.clone(),
736 json: false,
737 resume: false,
738 retry_failed: false,
739 reset_stale_claims: false,
740 stale_claim_secs: 1800,
741 max_cost_usd: args.max_cost_usd,
742 llm_parallelism: args.llm_parallelism as u32,
743 wait_job_singleton: args.wait_job_singleton,
744 force_job_singleton: args.force_job_singleton,
745 names: Vec::new(),
746 names_file: None,
747 preflight_check: false,
748 fallback_mode: None,
749 rate_limit_buffer: 300,
750 max_load_check: true,
751 circuit_breaker_threshold: 5,
752 preserve_threshold: 0.7,
753 entity_description_grounding_threshold: 0.12,
754 force_redescribe: false,
755 quality_sample: None,
756 entity_names: Vec::new(),
757 memory_names: Vec::new(),
758 anchor_memory: None,
759 entity_description_domain: "auto".to_string(),
760 yield_every_n_items: None,
761 ops_gate: false,
762 codex_model_validate: true,
763 codex_model_fallback: None,
764 min_output_chars: 500,
765 max_output_chars: 2000,
766 preserve_check: true,
767 prompt_template: None,
768 until_empty: false,
769 max_runtime: None,
770 max_attempts: 5,
771 status: false,
772 rest_concurrency: None,
773 list_dead: false,
776 requeue_dead: false,
777 list_skipped: false,
778 requeue_skipped: false,
779 prune_dead_orphans: false,
780 prune_dead_entity_orphans: false,
781 ignore_backoff: false,
782 body_extract_graph_only: false,
783 print_schema: false,
784 };
785 match crate::commands::enrich::run(&enrich_args, llm_backend, embedding_backend) {
786 Ok(()) => {
787 output::emit_json_compact(&serde_json::json!({
788 "event": "enrich_phase_completed"
789 }))?;
790 }
791 Err(e) => {
792 tracing::warn!(error = %e, "enrich --operation memory-bindings failed after ingest");
793 output::emit_json_compact(&serde_json::json!({
794 "event": "enrich_phase_failed",
795 "error": e.to_string()
796 }))?;
797 }
798 }
799 }
800
801 Ok(())
802}