1use std::collections::{BTreeSet, HashMap};
14use std::fs;
15use std::path::{Path, PathBuf};
16
17use chrono::{DateTime, Utc};
18use khive_runtime::{KhiveRuntime, NamespaceToken, RuntimeError};
19use khive_storage::{Edge, Entity, LinkId};
20use khive_types::EdgeRelation;
21use serde_json::{json, Value};
22use uuid::Uuid;
23
24use crate::imports::{self, Resolved};
25use crate::ingest::CODE_INGEST_NAMESPACE;
26use crate::manifest;
27
28#[derive(Debug, Default, serde::Serialize)]
31pub struct CodeSourceIngestReport {
32 pub projects_created: u64,
33 pub projects_updated: u64,
34 pub modules_created: u64,
35 pub modules_updated: u64,
36 pub edges_created: u64,
37 pub edges_updated: u64,
38 pub unresolved_recorded: u64,
39 pub unresolved_resolved: u64,
40 pub languages: Vec<String>,
41 pub warnings: Vec<String>,
44 pub db_path: String,
45}
46
47#[derive(Debug, thiserror::Error)]
48pub enum CodeSourceIngestError {
49 #[error("path {0:?} does not exist or is not a directory")]
50 InvalidPath(PathBuf),
51 #[error("runtime error: {0}")]
52 Runtime(#[from] RuntimeError),
53 #[error("storage error: {0}")]
54 Storage(String),
55}
56
57pub struct CodeSourceIngestOptions<'a> {
58 pub path: &'a Path,
59 pub languages: BTreeSet<&'static str>,
60 pub sweep_time: DateTime<Utc>,
61}
62
63fn uuid5_json(value: &Value) -> Uuid {
64 let bytes = serde_json::to_vec(value).expect("Value always serializes");
65 Uuid::new_v5(&CODE_INGEST_NAMESPACE, &bytes)
66}
67
68fn project_uuid(source_project: &str) -> Uuid {
69 uuid5_json(&json!({
70 "kind": "code-source-project",
71 "source_project": source_project,
72 }))
73}
74
75fn module_uuid(source_project: &str, language: &str, module_path: &str) -> Uuid {
76 uuid5_json(&json!({
77 "kind": "code-source-symbol",
78 "source_project": source_project,
79 "language": language,
80 "module_path": module_path,
81 "name": module_path,
82 "symbol_kind": "module",
83 }))
84}
85
86fn edge_uuid(relation: EdgeRelation, source_id: Uuid, target_id: Uuid) -> Uuid {
97 uuid5_json(&json!({
98 "kind": "code-source-edge",
99 "relation": relation.as_str(),
100 "source_id": source_id.to_string(),
101 "target_id": target_id.to_string(),
102 }))
103}
104
105#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)]
109struct UnresolvedSpec {
110 specifier: String,
111 target_kind: String,
112 dependency_kind: String,
113 language: String,
114}
115
116fn read_unresolved(properties: &Value) -> Vec<UnresolvedSpec> {
117 properties
118 .get("unresolved_specifiers")
119 .and_then(|v| serde_json::from_value(v.clone()).ok())
120 .unwrap_or_default()
121}
122
123async fn get_entity_opt(
124 rt: &KhiveRuntime,
125 token: &NamespaceToken,
126 id: Uuid,
127) -> Result<Option<Entity>, CodeSourceIngestError> {
128 rt.entities(token)?
129 .get_entity(id)
130 .await
131 .map_err(|e| CodeSourceIngestError::Storage(e.to_string()))
132}
133
134async fn upsert_entity(
135 rt: &KhiveRuntime,
136 token: &NamespaceToken,
137 entity: Entity,
138) -> Result<(), CodeSourceIngestError> {
139 rt.entities(token)?
140 .upsert_entity(entity)
141 .await
142 .map_err(|e| CodeSourceIngestError::Storage(e.to_string()))
143}
144
145#[allow(clippy::too_many_arguments)]
149async fn upsert_edge(
150 rt: &KhiveRuntime,
151 token: &NamespaceToken,
152 id: Uuid,
153 source_id: Uuid,
154 target_id: Uuid,
155 relation: EdgeRelation,
156 metadata: Value,
157 now: DateTime<Utc>,
158) -> Result<bool, CodeSourceIngestError> {
159 let link_id = LinkId::from(id);
160 let graph = rt.graph(token)?;
161 let existed = graph
162 .get_edge(link_id)
163 .await
164 .map_err(|e| CodeSourceIngestError::Storage(e.to_string()))?
165 .is_some();
166 let edge = Edge {
167 id: link_id,
168 namespace: token.namespace().as_str().to_string(),
169 source_id,
170 target_id,
171 relation,
172 weight: 1.0,
173 created_at: now,
174 updated_at: now,
175 deleted_at: None,
176 metadata: Some(metadata),
177 target_backend: None,
178 };
179 graph
180 .upsert_edge(edge)
181 .await
182 .map_err(|e| CodeSourceIngestError::Storage(e.to_string()))?;
183 Ok(!existed)
184}
185
186fn ts(dt: DateTime<Utc>) -> i64 {
187 dt.timestamp_micros()
188}
189
190async fn upsert_project(
194 rt: &KhiveRuntime,
195 token: &NamespaceToken,
196 name: &str,
197 language: &str,
198 sweep_time: DateTime<Utc>,
199 report: &mut CodeSourceIngestReport,
200) -> Result<Uuid, CodeSourceIngestError> {
201 let id = project_uuid(name);
202 let existing = get_entity_opt(rt, token, id).await?;
203 let is_new = existing.is_none();
204
205 let mut sweep_clock = existing
206 .as_ref()
207 .and_then(|e| e.properties.as_ref())
208 .and_then(|p| p.get("sweep_clock"))
209 .and_then(|v| v.as_object().cloned())
210 .unwrap_or_default();
211 sweep_clock.insert(language.to_string(), json!(sweep_time.to_rfc3339()));
212
213 let unresolved = existing
214 .as_ref()
215 .and_then(|e| e.properties.as_ref())
216 .map(read_unresolved)
217 .unwrap_or_default();
218
219 let mut props = serde_json::Map::new();
220 props.insert("source_project".into(), json!(name));
221 props.insert("last_seen_at".into(), json!(sweep_time.to_rfc3339()));
222 props.insert("sweep_clock".into(), Value::Object(sweep_clock));
223 if !unresolved.is_empty() {
224 props.insert(
225 "unresolved_specifiers".into(),
226 serde_json::to_value(&unresolved).expect("serializes"),
227 );
228 }
229
230 let mut entity = Entity::new(token.namespace().as_str(), "project", name);
231 entity.id = id;
232 entity.properties = Some(Value::Object(props));
233 let now = ts(sweep_time);
234 entity.created_at = existing.as_ref().map(|e| e.created_at).unwrap_or(now);
235 entity.updated_at = now;
236 upsert_entity(rt, token, entity).await?;
237
238 if is_new {
239 report.projects_created += 1;
240 } else {
241 report.projects_updated += 1;
242 }
243 Ok(id)
244}
245
246#[allow(clippy::too_many_arguments)]
247async fn upsert_module(
248 rt: &KhiveRuntime,
249 token: &NamespaceToken,
250 source_project: &str,
251 language: &str,
252 module_path: &str,
253 content_hash: &str,
254 sweep_time: DateTime<Utc>,
255 report: &mut CodeSourceIngestReport,
256) -> Result<Uuid, CodeSourceIngestError> {
257 let id = module_uuid(source_project, language, module_path);
258 let existing = get_entity_opt(rt, token, id).await?;
259 let is_new = existing.is_none();
260
261 let unresolved = existing
262 .as_ref()
263 .and_then(|e| e.properties.as_ref())
264 .map(read_unresolved)
265 .unwrap_or_default();
266
267 let mut props = serde_json::Map::new();
268 props.insert("source_project".into(), json!(source_project));
269 props.insert("language".into(), json!(language));
270 props.insert("module_path".into(), json!(module_path));
271 props.insert("content_hash".into(), json!(content_hash));
272 props.insert("last_seen_at".into(), json!(sweep_time.to_rfc3339()));
273 if !unresolved.is_empty() {
274 props.insert(
275 "unresolved_specifiers".into(),
276 serde_json::to_value(&unresolved).expect("serializes"),
277 );
278 }
279
280 let mut entity = Entity::new(token.namespace().as_str(), "concept", module_path)
281 .with_entity_type(Some("module"));
282 entity.id = id;
283 entity.properties = Some(Value::Object(props));
284 let now = ts(sweep_time);
285 entity.created_at = existing.as_ref().map(|e| e.created_at).unwrap_or(now);
286 entity.updated_at = now;
287 upsert_entity(rt, token, entity).await?;
288
289 if is_new {
290 report.modules_created += 1;
291 } else {
292 report.modules_updated += 1;
293 }
294 Ok(id)
295}
296
297async fn record_unresolved(
302 rt: &KhiveRuntime,
303 token: &NamespaceToken,
304 entity_id: Uuid,
305 spec: UnresolvedSpec,
306 report: &mut CodeSourceIngestReport,
307) -> Result<(), CodeSourceIngestError> {
308 let Some(mut entity) = get_entity_opt(rt, token, entity_id).await? else {
309 return Ok(());
310 };
311 let mut list = entity
312 .properties
313 .as_ref()
314 .map(read_unresolved)
315 .unwrap_or_default();
316 if list.contains(&spec) {
317 return Ok(());
318 }
319 list.push(spec);
320 report.unresolved_recorded += 1;
321 let mut props = entity
322 .properties
323 .clone()
324 .and_then(|v| v.as_object().cloned())
325 .unwrap_or_default();
326 props.insert(
327 "unresolved_specifiers".into(),
328 serde_json::to_value(&list).expect("serializes"),
329 );
330 entity.properties = Some(Value::Object(props));
331 upsert_entity(rt, token, entity).await
332}
333
334fn module_path_separator(language: &str) -> &'static str {
337 match language {
338 "python" => ".",
339 "typescript" => "/",
340 _ => "::",
341 }
342}
343
344fn module_candidate_specifiers(language: &str, specifier: &str) -> Vec<String> {
355 let sep = module_path_separator(language);
356 let segments: Vec<&str> = specifier.split(sep).filter(|s| !s.is_empty()).collect();
357 if segments.is_empty() {
358 return vec![specifier.to_string()];
359 }
360 (1..=segments.len())
361 .rev()
362 .map(|n| segments[..n].join(sep))
363 .collect()
364}
365
366fn target_ids_for(source_project: &str, spec: &UnresolvedSpec) -> Vec<Uuid> {
370 match spec.target_kind.as_str() {
371 "module" => module_candidate_specifiers(&spec.language, &spec.specifier)
372 .into_iter()
373 .map(|path| module_uuid(source_project, &spec.language, &path))
374 .collect(),
375 _ => vec![project_uuid(&spec.specifier)],
376 }
377}
378
379fn merge_dependency_kinds(
383 existing_metadata: Option<&Value>,
384 new_kind: &str,
385 language: &str,
386) -> Value {
387 let mut kinds: BTreeSet<String> = existing_metadata
388 .and_then(|m| m.get("dependency_kinds"))
389 .and_then(|v| v.as_array())
390 .map(|arr| {
391 arr.iter()
392 .filter_map(|v| v.as_str().map(str::to_string))
393 .collect()
394 })
395 .unwrap_or_default();
396 kinds.insert(new_kind.to_string());
397 json!({
398 "dependency_kinds": kinds.into_iter().collect::<Vec<_>>(),
399 "language": language,
400 })
401}
402
403#[allow(clippy::too_many_arguments)]
411async fn upsert_dependency_edge(
412 rt: &KhiveRuntime,
413 token: &NamespaceToken,
414 source_id: Uuid,
415 target_id: Uuid,
416 dependency_kind: &str,
417 language: &str,
418 now: DateTime<Utc>,
419 report: &mut CodeSourceIngestReport,
420) -> Result<(), CodeSourceIngestError> {
421 let edge_id = edge_uuid(EdgeRelation::DependsOn, source_id, target_id);
422 let link_id = LinkId::from(edge_id);
423 let graph = rt.graph(token)?;
424 let existing = graph
425 .get_edge(link_id)
426 .await
427 .map_err(|e| CodeSourceIngestError::Storage(e.to_string()))?;
428 let existed = existing.is_some();
429 let metadata = merge_dependency_kinds(
430 existing.as_ref().and_then(|e| e.metadata.as_ref()),
431 dependency_kind,
432 language,
433 );
434 let edge = Edge {
435 id: link_id,
436 namespace: token.namespace().as_str().to_string(),
437 source_id,
438 target_id,
439 relation: EdgeRelation::DependsOn,
440 weight: 1.0,
441 created_at: existing.as_ref().map(|e| e.created_at).unwrap_or(now),
442 updated_at: now,
443 deleted_at: None,
444 metadata: Some(metadata),
445 target_backend: None,
446 };
447 graph
448 .upsert_edge(edge)
449 .await
450 .map_err(|e| CodeSourceIngestError::Storage(e.to_string()))?;
451 if existed {
452 report.edges_updated += 1;
453 } else {
454 report.edges_created += 1;
455 }
456 Ok(())
457}
458
459async fn reresolve_pass(
464 rt: &KhiveRuntime,
465 token: &NamespaceToken,
466 now: DateTime<Utc>,
467 report: &mut CodeSourceIngestReport,
468) -> Result<(), CodeSourceIngestError> {
469 use khive_storage::types::{SqlStatement, SqlValue};
470
471 let sql = rt.sql();
472 let mut reader = sql
473 .reader()
474 .await
475 .map_err(|e| CodeSourceIngestError::Storage(e.to_string()))?;
476 let rows = reader
477 .query_all(SqlStatement {
478 sql: "SELECT id, kind, properties FROM entities WHERE namespace=?1 \
479 AND deleted_at IS NULL \
480 AND json_extract(properties,'$.unresolved_specifiers') IS NOT NULL"
481 .into(),
482 params: vec![SqlValue::Text(token.namespace().as_str().to_string())],
483 label: Some("code_ingest_reresolve_scan".into()),
484 })
485 .await
486 .map_err(|e| CodeSourceIngestError::Storage(e.to_string()))?;
487
488 for row in rows {
489 let id = match row.get("id") {
490 Some(SqlValue::Uuid(u)) => *u,
491 Some(SqlValue::Text(s)) => match Uuid::parse_str(s) {
492 Ok(u) => u,
493 Err(_) => continue,
494 },
495 _ => continue,
496 };
497 let Some(mut entity) = get_entity_opt(rt, token, id).await? else {
498 continue;
499 };
500 let source_project = entity
501 .properties
502 .as_ref()
503 .and_then(|p| p.get("source_project"))
504 .and_then(Value::as_str)
505 .unwrap_or(entity.name.as_str())
506 .to_string();
507 let mut list = entity
508 .properties
509 .as_ref()
510 .map(read_unresolved)
511 .unwrap_or_default();
512 if list.is_empty() {
513 continue;
514 }
515 let mut still_unresolved = Vec::new();
516 let mut changed = false;
517 for spec in list.drain(..) {
518 let mut resolved_target = None;
519 for target_id in target_ids_for(&source_project, &spec) {
520 if get_entity_opt(rt, token, target_id).await?.is_some() {
521 resolved_target = Some(target_id);
522 break;
523 }
524 }
525 match resolved_target {
526 Some(target_id) => {
527 upsert_dependency_edge(
528 rt,
529 token,
530 entity.id,
531 target_id,
532 &spec.dependency_kind,
533 &spec.language,
534 now,
535 report,
536 )
537 .await?;
538 report.unresolved_resolved += 1;
539 changed = true;
540 }
541 None => still_unresolved.push(spec),
542 }
543 }
544 if changed {
545 let mut props = entity
546 .properties
547 .clone()
548 .and_then(|v| v.as_object().cloned())
549 .unwrap_or_default();
550 if still_unresolved.is_empty() {
551 props.remove("unresolved_specifiers");
552 } else {
553 props.insert(
554 "unresolved_specifiers".into(),
555 serde_json::to_value(&still_unresolved).expect("serializes"),
556 );
557 }
558 entity.properties = Some(Value::Object(props));
559 upsert_entity(rt, token, entity).await?;
560 }
561 }
562 Ok(())
563}
564
565fn collect_source_files(root: &Path, ext: &str, out: &mut Vec<PathBuf>) -> std::io::Result<()> {
566 const SKIP_DIRS: &[&str] = &[
567 ".git",
568 "target",
569 "node_modules",
570 "__pycache__",
571 ".venv",
572 "venv",
573 "dist",
574 "build",
575 ];
576 for entry in fs::read_dir(root)? {
577 let entry = entry?;
578 let path = entry.path();
579 let file_type = entry.file_type()?;
580 if file_type.is_dir() {
581 let name = entry.file_name();
582 let name = name.to_string_lossy();
583 if SKIP_DIRS.contains(&name.as_ref()) || name.starts_with('.') {
584 continue;
585 }
586 collect_source_files(&path, ext, out)?;
587 } else if path.extension().and_then(|e| e.to_str()) == Some(ext) {
588 out.push(path);
589 }
590 }
591 Ok(())
592}
593
594fn content_hash(content: &str) -> String {
595 let mut hash: u64 = 0xcbf29ce484222325;
598 for b in content.as_bytes() {
599 hash ^= *b as u64;
600 hash = hash.wrapping_mul(0x100000001b3);
601 }
602 format!("{hash:016x}")
603}
604
605pub async fn run_code_ingest(
609 rt: &KhiveRuntime,
610 token: &NamespaceToken,
611 opts: CodeSourceIngestOptions<'_>,
612) -> Result<CodeSourceIngestReport, CodeSourceIngestError> {
613 if !opts.path.is_dir() {
614 return Err(CodeSourceIngestError::InvalidPath(opts.path.to_path_buf()));
615 }
616
617 let mut report = CodeSourceIngestReport {
618 languages: opts.languages.iter().map(|s| s.to_string()).collect(),
619 ..Default::default()
620 };
621
622 let manifests = manifest::discover_manifests(opts.path, &opts.languages)
623 .map_err(|e| CodeSourceIngestError::InvalidPath(opts.path.join(e.to_string())))?;
624
625 let mut project_ids: HashMap<String, Uuid> = HashMap::new();
626 for m in &manifests {
627 let id =
628 upsert_project(rt, token, &m.name, m.language, opts.sweep_time, &mut report).await?;
629 project_ids.insert(m.name.clone(), id);
630 }
631
632 for m in &manifests {
634 let source_id = project_ids[&m.name];
635 for (dep_name, dep_kind) in &m.dependencies {
636 let spec = UnresolvedSpec {
637 specifier: dep_name.clone(),
638 target_kind: "project".to_string(),
639 dependency_kind: dep_kind.clone(),
640 language: m.language.to_string(),
641 };
642 record_unresolved(rt, token, source_id, spec, &mut report).await?;
643 }
644 }
645
646 for language in opts.languages.iter().copied() {
653 run_import_scan(
654 rt,
655 token,
656 language,
657 opts.path,
658 opts.sweep_time,
659 &mut project_ids,
660 &mut report,
661 )
662 .await?;
663 }
664
665 reresolve_pass(rt, token, opts.sweep_time, &mut report).await?;
666
667 Ok(report)
668}
669
670fn basename_project_name(ingest_root: &Path) -> String {
673 ingest_root
674 .file_name()
675 .map(|n| n.to_string_lossy().to_string())
676 .unwrap_or_else(|| ingest_root.display().to_string())
677}
678
679#[allow(clippy::too_many_arguments)]
680async fn run_import_scan(
681 rt: &KhiveRuntime,
682 token: &NamespaceToken,
683 language: &'static str,
684 ingest_root: &Path,
685 sweep_time: DateTime<Utc>,
686 project_ids: &mut HashMap<String, Uuid>,
687 report: &mut CodeSourceIngestReport,
688) -> Result<(), CodeSourceIngestError> {
689 let Some(ext) = imports::extension_for_language(language) else {
690 return Ok(());
691 };
692 let mut files = Vec::new();
693 if let Err(e) = collect_source_files(ingest_root, ext, &mut files) {
694 report
695 .warnings
696 .push(format!("walking {}: {e}", ingest_root.display()));
697 return Ok(());
698 }
699
700 for file in files {
701 let Some(file_dir) = file.parent() else {
702 continue;
703 };
704 let (proj_root, proj_name) =
705 manifest::find_governing_manifest(file_dir, ingest_root, language).unwrap_or_else(
706 || {
707 (
708 ingest_root.to_path_buf(),
709 basename_project_name(ingest_root),
710 )
711 },
712 );
713 let Some(module_path) = imports::module_path_for_file(&file, &proj_root, language) else {
714 continue;
715 };
716
717 let proj_id = match project_ids.get(&proj_name) {
718 Some(id) => *id,
719 None => {
720 let id =
721 upsert_project(rt, token, &proj_name, language, sweep_time, report).await?;
722 project_ids.insert(proj_name.clone(), id);
723 id
724 }
725 };
726
727 let content = match fs::read_to_string(&file) {
728 Ok(c) => c,
729 Err(e) => {
730 report
731 .warnings
732 .push(format!("reading {}: {e}", file.display()));
733 continue;
734 }
735 };
736 let hash = content_hash(&content);
737 let module_id = upsert_module(
738 rt,
739 token,
740 &proj_name,
741 language,
742 &module_path,
743 &hash,
744 sweep_time,
745 report,
746 )
747 .await?;
748
749 let contains_edge_id = edge_uuid(EdgeRelation::Contains, proj_id, module_id);
750 let contains_created = upsert_edge(
751 rt,
752 token,
753 contains_edge_id,
754 proj_id,
755 module_id,
756 EdgeRelation::Contains,
757 json!({}),
758 sweep_time,
759 )
760 .await?;
761 if contains_created {
762 report.edges_created += 1;
763 } else {
764 report.edges_updated += 1;
765 }
766
767 for raw in imports::extract_raw_imports(language, &content) {
768 let resolved = if language == "typescript" && raw.starts_with('.') {
769 let rel_dir = file_dir.strip_prefix(&proj_root).unwrap_or(Path::new(""));
770 Resolved::IntraModule(imports::resolve_relative_ts_module(rel_dir, &raw))
771 } else {
772 imports::classify_import(language, &raw, &module_path, &proj_name)
773 };
774 match resolved {
775 Resolved::Skip => {}
776 Resolved::IntraModule(target_module_path) => {
777 let spec = UnresolvedSpec {
778 specifier: target_module_path,
779 target_kind: "module".to_string(),
780 dependency_kind: "import".to_string(),
781 language: language.to_string(),
782 };
783 record_unresolved(rt, token, module_id, spec, report).await?;
784 }
785 Resolved::ExternalProject(target_name) => {
786 let spec = UnresolvedSpec {
787 specifier: target_name,
788 target_kind: "project".to_string(),
789 dependency_kind: "import".to_string(),
790 language: language.to_string(),
791 };
792 record_unresolved(rt, token, proj_id, spec, report).await?;
793 }
794 }
795 }
796 }
797 Ok(())
798}