1use std::collections::HashMap;
12use std::path::{Path, PathBuf};
13use std::process::Command;
14
15use anyhow::{anyhow, Context, Result};
16use chrono::Utc;
17use serde::{Deserialize, Serialize};
18use serde_json::json;
19use uuid::Uuid;
20
21use khive_runtime::{secret_gate, KhiveRuntime, NamespaceToken, VerbRegistry};
22use khive_storage::types::{SqlStatement, SqlValue};
23
24use crate::hook;
25use crate::refs;
26
27#[derive(Debug, Clone, Copy)]
31pub struct IngestInclude {
32 pub commits: bool,
33 pub issues: bool,
34 pub pull_requests: bool,
35}
36
37impl Default for IngestInclude {
38 fn default() -> Self {
39 Self {
40 commits: true,
41 issues: true,
42 pull_requests: true,
43 }
44 }
45}
46
47#[derive(Debug, Clone)]
49pub struct IngestOptions {
50 pub repo: PathBuf,
52 pub project: String,
54 pub max_items: Option<u64>,
58 pub include: IngestInclude,
60}
61
62impl IngestOptions {
63 pub fn unbounded(repo: PathBuf, project: String) -> Self {
66 Self {
67 repo,
68 project,
69 max_items: None,
70 include: IngestInclude::default(),
71 }
72 }
73}
74
75struct Budget {
80 remaining: Option<u64>,
81}
82
83impl Budget {
84 fn try_consume(&mut self) -> bool {
85 match &mut self.remaining {
86 None => true,
87 Some(0) => false,
88 Some(n) => {
89 *n -= 1;
90 true
91 }
92 }
93 }
94
95 fn exhausted(&self) -> bool {
96 matches!(self.remaining, Some(0))
97 }
98}
99
100struct NewRecordForRef {
105 id: Uuid,
106 text: String,
107}
108
109#[derive(Debug, Default, Serialize)]
111pub struct IngestReport {
112 pub commits_ingested: u64,
113 pub commits_skipped_existing: u64,
114 pub issues_ingested: u64,
115 pub issues_skipped_existing: u64,
116 pub prs_ingested: u64,
117 pub prs_skipped_existing: u64,
118 pub gh_available: bool,
121 pub warnings: Vec<String>,
122 pub done: bool,
127 pub project_id: Option<String>,
130 pub project_created: bool,
134 pub reference_edges_created: u64,
138 pub reference_edges_unresolved: u64,
142 pub parent_edges_created: u64,
145 pub commit_embeddings_truncated: u64,
150}
151
152pub async fn run_ingest(
165 runtime: &KhiveRuntime,
166 token: &NamespaceToken,
167 registry: &VerbRegistry,
168 opts: IngestOptions,
169) -> Result<IngestReport> {
170 run_ingest_with_commit_recovery(runtime, token, registry, opts, |_repo, _err| Ok(None)).await
171}
172
173pub(crate) async fn run_ingest_with_commit_recovery(
184 runtime: &KhiveRuntime,
185 token: &NamespaceToken,
186 registry: &VerbRegistry,
187 opts: IngestOptions,
188 mut recover: impl FnMut(&Path, &GitLogError) -> Result<Option<RecoveredRepo>> + Send,
189) -> Result<IngestReport> {
190 let mut report = IngestReport {
191 done: true,
192 ..IngestReport::default()
193 };
194
195 let project_id = resolve_id(runtime, token, &opts.project)
196 .await?
197 .ok_or_else(|| anyhow!("--project {:?} did not resolve to an entity", opts.project))?;
198 report.project_id = Some(project_id.to_string());
199
200 let mut merge_sha_to_pr: HashMap<String, Uuid> = HashMap::new();
201 let mut number_to_pr: HashMap<u64, Uuid> = HashMap::new();
202 let mut budget = Budget {
203 remaining: opts.max_items,
204 };
205 let mut new_records: Vec<NewRecordForRef> = Vec::new();
206
207 if opts.include.issues || opts.include.pull_requests {
213 if gh_available(&opts.repo) {
214 report.gh_available = true;
215 if opts.include.pull_requests && !budget.exhausted() {
216 match ingest_prs(
217 runtime,
218 token,
219 registry,
220 &opts.repo,
221 project_id,
222 &mut report,
223 &mut merge_sha_to_pr,
224 &mut number_to_pr,
225 &mut budget,
226 &mut new_records,
227 )
228 .await
229 {
230 Ok(()) => {}
231 Err(e) => report
232 .warnings
233 .push(format!("gh pr list failed, skipping pull requests: {e}")),
234 }
235 }
236 if opts.include.issues && !budget.exhausted() {
237 if let Err(e) = ingest_issues(
238 runtime,
239 token,
240 registry,
241 &opts.repo,
242 project_id,
243 &mut report,
244 &mut budget,
245 &mut new_records,
246 )
247 .await
248 {
249 report
250 .warnings
251 .push(format!("gh issue list failed, skipping issues: {e}"));
252 }
253 }
254 } else {
255 report.gh_available = false;
256 report.warnings.push(
257 "gh CLI not found on PATH; skipped issues and pull requests — commits still ingest"
258 .to_string(),
259 );
260 }
261 }
262
263 if opts.include.commits && !budget.exhausted() {
264 ingest_commits(
265 runtime,
266 token,
267 registry,
268 &opts.repo,
269 project_id,
270 &merge_sha_to_pr,
271 &number_to_pr,
272 &mut report,
273 &mut budget,
274 &mut new_records,
275 &mut recover,
276 )
277 .await?;
278 }
279
280 if budget.exhausted() {
281 report.done = false;
282 }
283
284 link_references(
285 runtime,
286 token,
287 registry,
288 project_id,
289 &new_records,
290 &mut report,
291 )
292 .await;
293
294 Ok(report)
295}
296
297async fn resolve_id(
300 runtime: &KhiveRuntime,
301 _token: &NamespaceToken,
302 raw: &str,
303) -> Result<Option<Uuid>> {
304 if let Ok(u) = Uuid::parse_str(raw) {
305 return Ok(Some(u));
306 }
307 runtime
308 .resolve_prefix_unfiltered(raw)
309 .await
310 .map_err(|e| anyhow!("{e}"))
311}
312
313pub async fn resolve_project_id(runtime: &KhiveRuntime, raw: &str) -> Result<Option<Uuid>> {
317 if let Ok(u) = Uuid::parse_str(raw) {
318 return Ok(Some(u));
319 }
320 runtime
321 .resolve_prefix_unfiltered(raw)
322 .await
323 .map_err(|e| anyhow!("{e}"))
324}
325
326async fn find_issue_or_pr_by_number(
330 runtime: &KhiveRuntime,
331 token: &NamespaceToken,
332 project_id: Uuid,
333 number: u64,
334) -> Result<Option<Uuid>> {
335 if let Some(id) = find_by_number(runtime, token, "issue", project_id, number).await? {
336 return Ok(Some(id));
337 }
338 find_by_number(runtime, token, "pull_request", project_id, number).await
339}
340
341async fn link_references(
349 runtime: &KhiveRuntime,
350 token: &NamespaceToken,
351 registry: &VerbRegistry,
352 project_id: Uuid,
353 new_records: &[NewRecordForRef],
354 report: &mut IngestReport,
355) {
356 for record in new_records {
357 let mentions = refs::dedupe_prefer_closes(refs::extract_references(&record.text));
358 for mention in mentions {
359 let target = match find_issue_or_pr_by_number(
360 runtime,
361 token,
362 project_id,
363 mention.number,
364 )
365 .await
366 {
367 Ok(Some(id)) => id,
368 Ok(None) => {
369 report.reference_edges_unresolved += 1;
370 continue;
371 }
372 Err(e) => {
373 report
374 .warnings
375 .push(format!("resolving reference #{}: {e}", mention.number));
376 continue;
377 }
378 };
379 if target == record.id {
380 continue;
383 }
384 match registry
385 .dispatch(
386 "link",
387 json!({
388 "source_id": record.id.to_string(),
389 "target_id": target.to_string(),
390 "relation": "annotates",
391 "metadata": { "ref_kind": mention.kind.as_str() },
392 }),
393 )
394 .await
395 {
396 Ok(_) => report.reference_edges_created += 1,
397 Err(e) => report.warnings.push(format!(
398 "linking reference #{} from {}: {e}",
399 mention.number, record.id
400 )),
401 }
402 }
403 }
404}
405
406fn gh_available(repo: &Path) -> bool {
409 Command::new("gh")
410 .arg("--version")
411 .current_dir(repo)
412 .output()
413 .map(|o| o.status.success())
414 .unwrap_or(false)
415}
416
417async fn find_commit_by_sha(
421 runtime: &KhiveRuntime,
422 token: &NamespaceToken,
423 sha: &str,
424) -> Result<Option<Uuid>> {
425 let sql = runtime.sql();
426 let mut r = sql.reader().await.map_err(|e| anyhow!("{e}"))?;
427 let row = r
428 .query_row(SqlStatement {
429 sql: "SELECT id FROM notes WHERE kind='commit' AND namespace=?1 \
430 AND deleted_at IS NULL AND json_extract(properties,'$.sha')=?2 LIMIT 1"
431 .into(),
432 params: vec![
433 SqlValue::Text(token.namespace().as_str().to_string()),
434 SqlValue::Text(sha.to_string()),
435 ],
436 label: Some("git_ingest_find_commit_by_sha".into()),
437 })
438 .await
439 .map_err(|e| anyhow!("{e}"))?;
440 Ok(row.and_then(|r| row_uuid(&r)))
441}
442
443async fn find_by_number(
448 runtime: &KhiveRuntime,
449 token: &NamespaceToken,
450 kind: &str,
451 project_id: Uuid,
452 number: u64,
453) -> Result<Option<Uuid>> {
454 let sql = runtime.sql();
455 let mut r = sql.reader().await.map_err(|e| anyhow!("{e}"))?;
456 let row = r
457 .query_row(SqlStatement {
458 sql: "SELECT id FROM notes WHERE kind=?1 AND namespace=?2 \
459 AND deleted_at IS NULL AND json_extract(properties,'$.number')=?3 \
460 AND json_extract(properties,'$.project_id')=?4 LIMIT 1"
461 .into(),
462 params: vec![
463 SqlValue::Text(kind.to_string()),
464 SqlValue::Text(token.namespace().as_str().to_string()),
465 SqlValue::Integer(number as i64),
466 SqlValue::Text(project_id.to_string()),
467 ],
468 label: Some("git_ingest_find_by_number".into()),
469 })
470 .await
471 .map_err(|e| anyhow!("{e}"))?;
472 Ok(row.and_then(|r| row_uuid(&r)))
473}
474
475fn row_uuid(row: &khive_storage::types::SqlRow) -> Option<Uuid> {
476 match row.get("id") {
477 Some(SqlValue::Uuid(u)) => Some(*u),
478 Some(SqlValue::Text(s)) => Uuid::parse_str(s).ok(),
479 _ => None,
480 }
481}
482
483fn escape_like(input: &str) -> String {
487 let mut out = String::with_capacity(input.len());
488 for c in input.chars() {
489 if matches!(c, '\\' | '%' | '_') {
490 out.push('\\');
491 }
492 out.push(c);
493 }
494 out
495}
496
497async fn find_document_for_path(
507 runtime: &KhiveRuntime,
508 token: &NamespaceToken,
509 path: &str,
510) -> Result<Option<Uuid>> {
511 let file_name = Path::new(path)
512 .file_name()
513 .and_then(|f| f.to_str())
514 .unwrap_or(path);
515 let sql = runtime.sql();
516 let namespace = token.namespace().as_str().to_string();
517 let like_pattern = format!("%{}", escape_like(path));
518
519 let mut r = sql.reader().await.map_err(|e| anyhow!("{e}"))?;
520 let row = r
521 .query_row(SqlStatement {
522 sql: "SELECT id FROM entities WHERE kind='document' AND namespace=?1 \
523 AND deleted_at IS NULL \
524 AND (json_extract(properties,'$.source_uri')=?2 OR name=?3 \
525 OR json_extract(properties,'$.source_uri') LIKE ?4 ESCAPE '\\') \
526 ORDER BY CASE WHEN json_extract(properties,'$.source_uri')=?2 OR name=?3 \
527 THEN 0 ELSE 1 END, id \
528 LIMIT 1"
529 .into(),
530 params: vec![
531 SqlValue::Text(namespace),
532 SqlValue::Text(path.to_string()),
533 SqlValue::Text(file_name.to_string()),
534 SqlValue::Text(like_pattern),
535 ],
536 label: Some("git_ingest_find_document_for_path".into()),
537 })
538 .await
539 .map_err(|e| anyhow!("{e}"))?;
540 Ok(row.and_then(|r| row_uuid(&r)))
541}
542
543async fn read_cursor(
545 runtime: &KhiveRuntime,
546 project_id: Uuid,
547 kind: &str,
548) -> Result<Option<String>> {
549 let sql = runtime.sql();
550 let mut r = sql.reader().await.map_err(|e| anyhow!("{e}"))?;
551 let row = r
552 .query_row(SqlStatement {
553 sql: "SELECT cursor_value FROM git_mirror_cursor WHERE project_id=?1 AND kind=?2"
554 .into(),
555 params: vec![
556 SqlValue::Text(project_id.to_string()),
557 SqlValue::Text(kind.to_string()),
558 ],
559 label: Some("git_ingest_read_cursor".into()),
560 })
561 .await
562 .map_err(|e| anyhow!("{e}"))?;
563 Ok(row.and_then(|r| match r.get("cursor_value") {
564 Some(SqlValue::Text(s)) => Some(s.clone()),
565 _ => None,
566 }))
567}
568
569async fn write_cursor(
577 runtime: &KhiveRuntime,
578 project_id: Uuid,
579 kind: &str,
580 value: &str,
581) -> Result<()> {
582 let sql = runtime.sql();
583 let mut w = sql.writer().await.map_err(|e| anyhow!("{e}"))?;
584 w.execute(SqlStatement {
585 sql: "INSERT INTO git_mirror_cursor(project_id, kind, cursor_value, updated_at) \
586 VALUES(?1, ?2, ?3, ?4) \
587 ON CONFLICT(project_id, kind) DO UPDATE SET \
588 cursor_value=excluded.cursor_value, \
589 updated_at=excluded.updated_at"
590 .into(),
591 params: vec![
592 SqlValue::Text(project_id.to_string()),
593 SqlValue::Text(kind.to_string()),
594 SqlValue::Text(value.to_string()),
595 SqlValue::Integer(Utc::now().timestamp_micros()),
596 ],
597 label: Some("git_ingest_write_cursor".into()),
598 })
599 .await
600 .map_err(|e| anyhow!("{e}"))?;
601 Ok(())
602}
603
604const RECORD_SEP: char = '\u{1e}';
607const FIELD_SEP: char = '\u{1f}';
608
609struct RawCommit {
610 sha: String,
611 short_sha: String,
612 author: String,
613 author_email: String,
614 committed_at: String,
615 parents: Vec<String>,
616 subject: String,
617 body: String,
618}
619
620#[derive(Debug, Clone, Copy, PartialEq, Eq)]
626pub(crate) enum GitLogPhase {
627 Metadata,
628 TouchedFiles,
629}
630
631#[derive(Debug)]
636pub(crate) struct GitLogError {
637 phase: GitLogPhase,
638 stderr: String,
639}
640
641impl std::fmt::Display for GitLogError {
642 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
643 let cmd = match self.phase {
644 GitLogPhase::Metadata => "git log",
645 GitLogPhase::TouchedFiles => "git log --name-only",
646 };
647 write!(f, "{cmd} failed: {}", self.stderr)
648 }
649}
650
651impl std::error::Error for GitLogError {}
652
653impl GitLogError {
654 pub(crate) fn is_missing_promisor_object(&self) -> bool {
661 let lower = self.stderr.to_ascii_lowercase();
662 lower.contains("promisor")
663 && (lower.contains("not in the object database") || lower.contains("missing object"))
664 }
665}
666
667fn walk_commits(repo: &Path, since_sha: Option<&str>) -> Result<Vec<RawCommit>> {
671 let format = format!("%H{FIELD_SEP}%h{FIELD_SEP}%an{FIELD_SEP}%ae{FIELD_SEP}%cI{FIELD_SEP}%P{FIELD_SEP}%s{FIELD_SEP}%b{RECORD_SEP}");
676 let mut args = vec![
677 "log".to_string(),
678 "--reverse".to_string(),
679 format!("--pretty=format:{format}"),
680 ];
681 if let Some(sha) = since_sha {
682 args.push(format!("{sha}..HEAD"));
683 }
684 let output = Command::new("git")
685 .arg("-C")
686 .arg(repo)
687 .args(&args)
688 .output()
689 .context("spawning git log")?;
690 if !output.status.success() {
691 return Err(anyhow::Error::new(GitLogError {
692 phase: GitLogPhase::Metadata,
693 stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
694 }));
695 }
696 let text = String::from_utf8_lossy(&output.stdout);
697 let mut commits = Vec::new();
698 for record in text.split(RECORD_SEP) {
699 let record = record.trim_matches('\n');
700 if record.is_empty() {
701 continue;
702 }
703 let fields: Vec<&str> = record.splitn(8, FIELD_SEP).collect();
704 if fields.len() < 8 {
705 continue;
706 }
707 let sha = fields[0].to_string();
708 let short_sha = fields[1].to_string();
709 let author = fields[2].to_string();
710 let author_email = fields[3].to_string();
711 let committed_at = fields[4].to_string();
712 let parents = fields[5]
713 .split_whitespace()
714 .map(str::to_string)
715 .collect::<Vec<_>>();
716 let subject = fields[6].to_string();
717 let body = fields[7].trim_end_matches('\n').to_string();
718 commits.push(RawCommit {
719 sha,
720 short_sha,
721 author,
722 author_email,
723 committed_at,
724 parents,
725 subject,
726 body,
727 });
728 }
729 Ok(commits)
730}
731
732fn touched_files(repo: &Path) -> Result<HashMap<String, Vec<String>>> {
737 let output = Command::new("git")
738 .arg("-C")
739 .arg(repo)
740 .arg("log")
741 .arg("--name-only")
742 .arg(format!("--pretty=format:{RECORD_SEP}%H"))
743 .output()
744 .context("spawning git log --name-only")?;
745 if !output.status.success() {
746 return Err(anyhow::Error::new(GitLogError {
747 phase: GitLogPhase::TouchedFiles,
748 stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
749 }));
750 }
751 let text = String::from_utf8_lossy(&output.stdout);
752 let mut map: HashMap<String, Vec<String>> = HashMap::new();
753 for block in text.split(RECORD_SEP) {
754 let mut lines = block.lines().filter(|l| !l.trim().is_empty());
755 let Some(sha) = lines.next() else { continue };
756 let files: Vec<String> = lines.map(str::to_string).collect();
757 map.insert(sha.trim().to_string(), files);
758 }
759 Ok(map)
760}
761
762struct CommitSnapshot {
766 commits: Vec<RawCommit>,
767 files_by_sha: HashMap<String, Vec<String>>,
768}
769
770fn load_commit_snapshot(repo: &Path, since_sha: Option<&str>) -> Result<CommitSnapshot> {
776 let commits = walk_commits(repo, since_sha)?;
777 if commits.is_empty() {
778 return Ok(CommitSnapshot {
779 commits,
780 files_by_sha: HashMap::new(),
781 });
782 }
783 let files_by_sha = touched_files(repo)?;
784 Ok(CommitSnapshot {
785 commits,
786 files_by_sha,
787 })
788}
789
790#[derive(Debug, Clone, Copy, PartialEq, Eq)]
794pub(crate) enum CacheRepairStrategy {
795 Refetch,
796 Reclone,
797}
798
799pub(crate) struct RecoveredRepo {
804 pub(crate) repo: PathBuf,
805 pub(crate) strategy: CacheRepairStrategy,
806}
807
808fn cache_repair_warning(strategy: CacheRepairStrategy) -> String {
809 match strategy {
810 CacheRepairStrategy::Refetch => {
811 "repaired corrupt remote git cache by refetching missing promisor objects".to_string()
812 }
813 CacheRepairStrategy::Reclone => {
814 "repaired corrupt remote git cache by replacing the owned clone".to_string()
815 }
816 }
817}
818
819fn recover_commit_snapshot(
832 repo: &Path,
833 since_sha: Option<&str>,
834 mut recover: impl FnMut(&Path, &GitLogError) -> Result<Option<RecoveredRepo>>,
835) -> Result<(CommitSnapshot, Option<String>)> {
836 let mut repo_path = repo.to_path_buf();
837 let mut recovery_warning: Option<String> = None;
838 loop {
839 match load_commit_snapshot(&repo_path, since_sha) {
840 Ok(snapshot) => return Ok((snapshot, recovery_warning)),
841 Err(e) => {
842 let classified = e
843 .downcast_ref::<GitLogError>()
844 .filter(|g| g.is_missing_promisor_object());
845 let Some(git_log_err) = classified else {
846 return Err(e);
847 };
848 match recover(&repo_path, git_log_err)? {
849 Some(recovered) => {
850 repo_path = recovered.repo;
851 recovery_warning = Some(cache_repair_warning(recovered.strategy));
852 }
853 None => return Err(e),
854 }
855 }
856 }
857 }
858}
859
860fn squash_merge_pr_number(subject: &str) -> Option<u64> {
862 let trimmed = subject.trim_end();
863 let close = trimmed.strip_suffix(')')?;
864 let open = close.rfind("(#")?;
865 close[open + 2..].parse::<u64>().ok()
866}
867
868const NAME_MAX_CHARS: usize = 120;
872
873const MAX_COMMIT_EMBED_BYTES: usize = 32_768;
879
880fn truncated_embedding_head(content: &str) -> Option<&str> {
884 if content.len() <= MAX_COMMIT_EMBED_BYTES {
885 return None;
886 }
887 let mut end = MAX_COMMIT_EMBED_BYTES;
888 while !content.is_char_boundary(end) {
889 end -= 1;
890 }
891 Some(&content[..end])
892}
893
894#[allow(clippy::too_many_arguments)]
895async fn ingest_commits(
896 runtime: &KhiveRuntime,
897 token: &NamespaceToken,
898 registry: &VerbRegistry,
899 repo: &Path,
900 project_id: Uuid,
901 merge_sha_to_pr: &HashMap<String, Uuid>,
902 number_to_pr: &HashMap<u64, Uuid>,
903 report: &mut IngestReport,
904 budget: &mut Budget,
905 new_records: &mut Vec<NewRecordForRef>,
906 recover: &mut (dyn FnMut(&Path, &GitLogError) -> Result<Option<RecoveredRepo>> + Send),
907) -> Result<()> {
908 let since = read_cursor(runtime, project_id, "commits").await?;
909 let (snapshot, recovery_warning) = recover_commit_snapshot(repo, since.as_deref(), recover)?;
910 let CommitSnapshot {
911 commits,
912 files_by_sha,
913 } = snapshot;
914 if commits.is_empty() {
915 if let Some(warning) = recovery_warning {
916 report.warnings.push(warning);
917 }
918 return Ok(());
919 }
920
921 let mut last_sha: Option<String> = since;
931 let mut cursor_stalled = false;
932 let mut local_sha_to_id: HashMap<String, Uuid> = HashMap::new();
937 for c in &commits {
938 if let Some(existing) = find_commit_by_sha(runtime, token, &c.sha).await? {
939 local_sha_to_id.insert(c.sha.clone(), existing);
940 report.commits_skipped_existing += 1;
941 if !cursor_stalled {
942 last_sha = Some(c.sha.clone());
943 }
944 continue;
945 }
946
947 if budget.exhausted() {
948 break;
949 }
950
951 let raw_content = if c.body.trim().is_empty() {
952 c.subject.clone()
953 } else {
954 format!("{}\n\n{}", c.subject, c.body)
955 };
956 let content = secret_gate::mask_secrets(&raw_content).into_owned();
957
958 let mut annotates = vec![project_id.to_string()];
959
960 if let Some(paths) = files_by_sha.get(&c.sha) {
961 for p in paths {
962 if !p.starts_with("docs/adr/") {
963 continue;
964 }
965 if let Some(doc_id) = find_document_for_path(runtime, token, p).await? {
966 annotates.push(doc_id.to_string());
967 }
968 }
969 }
970
971 let pr_id = match merge_sha_to_pr.get(&c.sha).copied() {
977 Some(id) => Some(id),
978 None => match squash_merge_pr_number(&c.subject) {
979 Some(n) => match number_to_pr.get(&n).copied() {
980 Some(id) => Some(id),
981 None => find_by_number(runtime, token, "pull_request", project_id, n).await?,
982 },
983 None => None,
984 },
985 };
986 if let Some(pr_id) = pr_id {
987 annotates.push(pr_id.to_string());
988 }
989
990 let properties = json!({
991 "sha": c.sha,
992 "short_sha": c.short_sha,
993 "author": c.author,
994 "author_email": c.author_email,
995 "committed_at": c.committed_at,
996 "parents": c.parents,
997 });
998
999 let name = refs::truncate_chars(&format!("{} {}", c.short_sha, c.subject), NAME_MAX_CHARS);
1000 let embedding_head = truncated_embedding_head(&content);
1001
1002 let mut create_request = json!({
1003 "kind": "commit",
1004 "name": name,
1005 "content": content,
1006 "properties": properties,
1007 "annotates": annotates,
1008 });
1009 if let Some(head) = embedding_head {
1010 create_request["embedding_content"] = json!(head);
1011 }
1012
1013 budget.try_consume();
1014 match registry.dispatch("create", create_request).await {
1015 Ok(v) => {
1016 report.commits_ingested += 1;
1017 if embedding_head.is_some() {
1018 report.commit_embeddings_truncated += 1;
1019 }
1020 if !cursor_stalled {
1021 last_sha = Some(c.sha.clone());
1022 }
1023 if let Some(id) = v
1024 .get("id")
1025 .and_then(|v| v.as_str())
1026 .and_then(|s| Uuid::parse_str(s).ok())
1027 {
1028 local_sha_to_id.insert(c.sha.clone(), id);
1029 new_records.push(NewRecordForRef {
1030 id,
1031 text: content.clone(),
1032 });
1033 for parent_sha in &c.parents {
1038 let parent_id = match local_sha_to_id.get(parent_sha).copied() {
1039 Some(pid) => Some(pid),
1040 None => find_commit_by_sha(runtime, token, parent_sha).await?,
1041 };
1042 let Some(parent_id) = parent_id else {
1043 continue;
1044 };
1045 if parent_id == id {
1046 continue;
1047 }
1048 match registry
1049 .dispatch(
1050 "link",
1051 json!({
1052 "source_id": parent_id.to_string(),
1053 "target_id": id.to_string(),
1054 "relation": "precedes",
1055 }),
1056 )
1057 .await
1058 {
1059 Ok(_) => report.parent_edges_created += 1,
1060 Err(e) => report.warnings.push(format!(
1061 "linking parent {parent_sha} -> {} precedes: {e}",
1062 c.sha
1063 )),
1064 }
1065 }
1066 }
1067 }
1068 Err(e) => {
1069 report
1070 .warnings
1071 .push(format!("create commit {}: {e}", c.sha));
1072 cursor_stalled = true;
1073 }
1074 }
1075 }
1076
1077 if let Some(sha) = last_sha {
1078 write_cursor(runtime, project_id, "commits", &sha).await?;
1079 }
1080 if let Some(warning) = recovery_warning {
1081 report.warnings.push(warning);
1082 }
1083 Ok(())
1084}
1085
1086#[derive(Debug, Deserialize)]
1089struct GhAuthor {
1090 login: Option<String>,
1091}
1092
1093#[derive(Debug, Deserialize)]
1094struct GhLabel {
1095 name: String,
1096}
1097
1098#[derive(Debug, Deserialize)]
1099struct GhIssue {
1100 number: u64,
1101 title: String,
1102 author: Option<GhAuthor>,
1103 #[serde(rename = "createdAt")]
1104 created_at: Option<String>,
1105 #[serde(rename = "closedAt")]
1106 closed_at: Option<String>,
1107 #[serde(rename = "updatedAt")]
1108 updated_at: Option<String>,
1109 labels: Option<Vec<GhLabel>>,
1110 #[serde(rename = "stateReason")]
1111 state_reason: Option<String>,
1112 body: Option<String>,
1113}
1114
1115#[derive(Debug, Deserialize)]
1116struct GhMergeCommit {
1117 oid: Option<String>,
1118}
1119
1120#[derive(Debug, Deserialize)]
1121struct GhPr {
1122 number: u64,
1123 title: String,
1124 author: Option<GhAuthor>,
1125 #[serde(rename = "createdAt")]
1126 created_at: Option<String>,
1127 #[serde(rename = "mergedAt")]
1128 merged_at: Option<String>,
1129 #[serde(rename = "closedAt")]
1130 closed_at: Option<String>,
1131 #[serde(rename = "updatedAt")]
1132 updated_at: Option<String>,
1133 #[serde(rename = "baseRefName")]
1134 base_ref_name: Option<String>,
1135 #[serde(rename = "headRefName")]
1136 head_ref_name: Option<String>,
1137 #[serde(rename = "mergeCommit")]
1138 merge_commit: Option<GhMergeCommit>,
1139 body: Option<String>,
1140}
1141
1142struct MaskedIssueFields {
1150 number: u64,
1151 title: String,
1152 body: String,
1153 author_login: Option<String>,
1154 labels: Vec<String>,
1155 created_at: Option<String>,
1156 closed_at: Option<String>,
1157 updated_at: Option<String>,
1158 state_reason: StateReasonField,
1159}
1160
1161#[derive(Debug, Clone, PartialEq, Eq)]
1168enum StateReasonField {
1169 Absent,
1170 Valid(String),
1171 Rejected,
1172}
1173
1174impl MaskedIssueFields {
1175 fn new(issue: GhIssue, warnings: &mut Vec<String>) -> Self {
1176 let GhIssue {
1177 number,
1178 title,
1179 author,
1180 created_at,
1181 closed_at,
1182 updated_at,
1183 labels,
1184 state_reason,
1185 body,
1186 } = issue;
1187
1188 Self {
1189 number,
1190 title: secret_gate::mask_secrets(&title).into_owned(),
1191 body: secret_gate::mask_secrets(&body.unwrap_or_default()).into_owned(),
1192 author_login: author
1193 .and_then(|a| a.login)
1194 .map(|login| secret_gate::mask_secrets(&login).into_owned()),
1195 labels: labels
1196 .unwrap_or_default()
1197 .into_iter()
1198 .map(|l| secret_gate::mask_secrets(&l.name).into_owned())
1199 .collect(),
1200 created_at: canonical_issue_timestamp("createdAt", number, created_at, warnings),
1201 closed_at: canonical_issue_timestamp("closedAt", number, closed_at, warnings),
1202 updated_at: canonical_issue_timestamp("updatedAt", number, updated_at, warnings),
1203 state_reason: canonical_issue_state_reason(state_reason),
1204 }
1205 }
1206}
1207
1208fn canonical_issue_state_reason(raw: Option<String>) -> StateReasonField {
1217 let Some(raw) = raw.filter(|r| !r.is_empty()) else {
1218 return StateReasonField::Absent;
1219 };
1220 let lowered = raw.to_ascii_lowercase();
1221 if hook::ISSUE_STATE_REASONS.contains(&lowered.as_str()) {
1222 StateReasonField::Valid(lowered)
1223 } else {
1224 StateReasonField::Rejected
1225 }
1226}
1227
1228fn canonical_issue_timestamp(
1238 field: &'static str,
1239 number: u64,
1240 raw: Option<String>,
1241 warnings: &mut Vec<String>,
1242) -> Option<String> {
1243 let raw = raw?;
1244 match chrono::DateTime::parse_from_rfc3339(&raw) {
1245 Ok(dt) => Some(
1246 dt.with_timezone(&Utc)
1247 .to_rfc3339_opts(chrono::SecondsFormat::Secs, true),
1248 ),
1249 Err(_) => {
1250 warnings.push(format!(
1251 "issue #{number}: {field} is not a valid RFC3339 timestamp, field dropped"
1252 ));
1253 None
1254 }
1255 }
1256}
1257
1258fn gh_json(repo: &Path, args: &[&str]) -> Result<String> {
1259 let output = Command::new("gh")
1261 .current_dir(repo)
1262 .args(args)
1263 .output()
1264 .context("spawning gh")?;
1265 if !output.status.success() {
1266 return Err(anyhow!(
1267 "gh {:?} failed: {}",
1268 args,
1269 String::from_utf8_lossy(&output.stderr)
1270 ));
1271 }
1272 Ok(String::from_utf8_lossy(&output.stdout).into_owned())
1273}
1274
1275const PAGE_LIMIT: usize = 1000;
1282
1283#[derive(Debug, Clone, PartialEq, Eq)]
1290enum PageOutcome {
1291 WindowComplete,
1294 StopBudgetExhausted,
1297 StopFloorStalled,
1302 Continue(String),
1305}
1306
1307fn decide_page_outcome(
1308 page_len: usize,
1309 current_floor: Option<&str>,
1310 last_updated_at: Option<&str>,
1311 budget_exhausted: bool,
1312) -> PageOutcome {
1313 if page_len < PAGE_LIMIT {
1314 return PageOutcome::WindowComplete;
1315 }
1316 if budget_exhausted {
1317 return PageOutcome::StopBudgetExhausted;
1318 }
1319 match last_updated_at {
1320 Some(next) if Some(next) != current_floor => PageOutcome::Continue(next.to_string()),
1321 _ => PageOutcome::StopFloorStalled,
1322 }
1323}
1324
1325#[cfg(test)]
1331fn page_outcome_proves_window_complete(outcome: PageOutcome) -> bool {
1332 matches!(outcome, PageOutcome::WindowComplete)
1333}
1334
1335fn search_query(floor: Option<&str>) -> String {
1336 match floor {
1337 Some(f) => format!("sort:updated-asc updated:>={f}"),
1338 None => "sort:updated-asc".to_string(),
1339 }
1340}
1341
1342const PR_FIELDS: &str = "number,title,author,createdAt,mergedAt,closedAt,updatedAt,baseRefName,headRefName,mergeCommit,body";
1343const ISSUE_FIELDS: &str =
1344 "number,title,author,createdAt,closedAt,updatedAt,labels,stateReason,body";
1345
1346fn fetch_pr_page(repo: &Path, floor: Option<&str>) -> Result<Vec<GhPr>> {
1347 let search = search_query(floor);
1348 let raw = gh_json(
1349 repo,
1350 &[
1351 "pr",
1352 "list",
1353 "--state",
1354 "all",
1355 "--search",
1356 search.as_str(),
1357 "--limit",
1358 "1000",
1359 "--json",
1360 PR_FIELDS,
1361 ],
1362 )?;
1363 serde_json::from_str(&raw).context("parsing gh pr list --json")
1364}
1365
1366fn fetch_issue_page(repo: &Path, floor: Option<&str>) -> Result<Vec<GhIssue>> {
1367 let search = search_query(floor);
1368 let raw = gh_json(
1369 repo,
1370 &[
1371 "issue",
1372 "list",
1373 "--state",
1374 "all",
1375 "--search",
1376 search.as_str(),
1377 "--limit",
1378 "1000",
1379 "--json",
1380 ISSUE_FIELDS,
1381 ],
1382 )?;
1383 serde_json::from_str(&raw).context("parsing gh issue list --json")
1384}
1385
1386#[allow(clippy::too_many_arguments)]
1387async fn ingest_prs(
1388 runtime: &KhiveRuntime,
1389 token: &NamespaceToken,
1390 registry: &VerbRegistry,
1391 repo: &Path,
1392 project_id: Uuid,
1393 report: &mut IngestReport,
1394 merge_sha_to_pr: &mut HashMap<String, Uuid>,
1395 number_to_pr: &mut HashMap<u64, Uuid>,
1396 budget: &mut Budget,
1397 new_records: &mut Vec<NewRecordForRef>,
1398) -> Result<()> {
1399 let since = read_cursor(runtime, project_id, "prs").await?;
1400
1401 let mut max_updated: Option<String> = since.clone();
1407 let mut cursor_stalled = false;
1408 let mut floor = since.clone();
1409 let mut window_complete = true;
1410
1411 'paging: loop {
1412 let mut page = fetch_pr_page(repo, floor.as_deref())?;
1413 let page_len = page.len();
1414 page.sort_by(|a, b| a.updated_at.cmp(&b.updated_at));
1423 let last_updated_at = page.last().and_then(|pr| pr.updated_at.clone());
1424
1425 for pr in page {
1426 let is_new = since
1427 .as_deref()
1428 .zip(pr.updated_at.as_deref())
1429 .map(|(cursor, updated)| updated >= cursor)
1430 .unwrap_or(true);
1431
1432 if let Some(existing) =
1433 find_by_number(runtime, token, "pull_request", project_id, pr.number).await?
1434 {
1435 number_to_pr.insert(pr.number, existing);
1436 if let Some(oid) = pr.merge_commit.as_ref().and_then(|m| m.oid.clone()) {
1437 merge_sha_to_pr.insert(oid, existing);
1438 }
1439 report.prs_skipped_existing += 1;
1440 if !cursor_stalled {
1441 if let Some(u) = &pr.updated_at {
1442 if max_updated
1443 .as_deref()
1444 .map(|m| u.as_str() > m)
1445 .unwrap_or(true)
1446 {
1447 max_updated = Some(u.clone());
1448 }
1449 }
1450 }
1451 continue;
1452 }
1453 if !is_new {
1454 continue;
1455 }
1456 if budget.exhausted() {
1457 break;
1458 }
1459
1460 let raw_body = pr.body.unwrap_or_default();
1461 let content = secret_gate::mask_secrets(&raw_body).into_owned();
1462 let safe_title = secret_gate::mask_secrets(&pr.title).into_owned();
1463 let properties = json!({
1464 "number": pr.number,
1465 "title": safe_title,
1466 "author": pr.author.and_then(|a| a.login),
1467 "created_at": pr.created_at,
1468 "merged_at": pr.merged_at,
1469 "closed_at": pr.closed_at,
1470 "base_ref": pr.base_ref_name,
1471 "head_ref": pr.head_ref_name,
1472 "project_id": project_id.to_string(),
1473 });
1474 let name =
1475 refs::truncate_chars(&format!("#{} {}", pr.number, safe_title), NAME_MAX_CHARS);
1476
1477 budget.try_consume();
1478 let result = match registry
1479 .dispatch(
1480 "create",
1481 json!({
1482 "kind": "pull_request",
1483 "name": name,
1484 "content": content,
1485 "properties": properties,
1486 "annotates": [project_id.to_string()],
1487 }),
1488 )
1489 .await
1490 {
1491 Ok(v) => v,
1492 Err(e) => {
1493 report
1494 .warnings
1495 .push(format!("create pull_request #{}: {e}", pr.number));
1496 cursor_stalled = true;
1497 continue;
1498 }
1499 };
1500
1501 if let Some(id) = result
1502 .get("id")
1503 .and_then(|v| v.as_str())
1504 .and_then(|s| Uuid::parse_str(s).ok())
1505 {
1506 number_to_pr.insert(pr.number, id);
1507 if let Some(oid) = pr.merge_commit.and_then(|m| m.oid) {
1508 merge_sha_to_pr.insert(oid, id);
1509 }
1510 new_records.push(NewRecordForRef {
1511 id,
1512 text: content.clone(),
1513 });
1514 }
1515 report.prs_ingested += 1;
1516 if !cursor_stalled {
1517 if let Some(u) = &pr.updated_at {
1518 if max_updated
1519 .as_deref()
1520 .map(|m| u.as_str() > m)
1521 .unwrap_or(true)
1522 {
1523 max_updated = Some(u.clone());
1524 }
1525 }
1526 }
1527 }
1528
1529 match decide_page_outcome(
1530 page_len,
1531 floor.as_deref(),
1532 last_updated_at.as_deref(),
1533 budget.exhausted(),
1534 ) {
1535 PageOutcome::WindowComplete => break 'paging,
1536 PageOutcome::StopBudgetExhausted | PageOutcome::StopFloorStalled => {
1537 window_complete = false;
1538 break 'paging;
1539 }
1540 PageOutcome::Continue(next_floor) => floor = Some(next_floor),
1541 }
1542 }
1543
1544 if !window_complete {
1545 report.done = false;
1550 }
1551
1552 if let Some(cursor) = max_updated {
1553 write_cursor(runtime, project_id, "prs", &cursor).await?;
1554 }
1555 Ok(())
1556}
1557
1558#[allow(clippy::too_many_arguments)]
1559async fn ingest_issues(
1560 runtime: &KhiveRuntime,
1561 token: &NamespaceToken,
1562 registry: &VerbRegistry,
1563 repo: &Path,
1564 project_id: Uuid,
1565 report: &mut IngestReport,
1566 budget: &mut Budget,
1567 new_records: &mut Vec<NewRecordForRef>,
1568) -> Result<()> {
1569 let since = read_cursor(runtime, project_id, "issues").await?;
1570
1571 let mut max_updated: Option<String> = since.clone();
1577 let mut cursor_stalled = false;
1578 let mut floor = since.clone();
1579 let mut window_complete = true;
1580
1581 'paging: loop {
1582 let page = fetch_issue_page(repo, floor.as_deref())?;
1583 let page_len = page.len();
1584 let mut masked_page: Vec<MaskedIssueFields> = page
1593 .into_iter()
1594 .map(|issue| MaskedIssueFields::new(issue, &mut report.warnings))
1595 .collect();
1596 masked_page.sort_by(|a, b| a.updated_at.cmp(&b.updated_at));
1601 let last_updated_at = masked_page.last().and_then(|i| i.updated_at.clone());
1602
1603 for masked in masked_page {
1604 let is_new = since
1605 .as_deref()
1606 .zip(masked.updated_at.as_deref())
1607 .map(|(cursor, updated)| updated >= cursor)
1608 .unwrap_or(true);
1609
1610 if find_by_number(runtime, token, "issue", project_id, masked.number)
1611 .await?
1612 .is_some()
1613 {
1614 report.issues_skipped_existing += 1;
1615 if !cursor_stalled {
1616 if let Some(u) = &masked.updated_at {
1617 if max_updated
1618 .as_deref()
1619 .map(|m| u.as_str() > m)
1620 .unwrap_or(true)
1621 {
1622 max_updated = Some(u.clone());
1623 }
1624 }
1625 }
1626 continue;
1627 }
1628 if !is_new {
1629 continue;
1630 }
1631 if budget.exhausted() {
1632 break;
1633 }
1634
1635 let number = masked.number;
1636 let updated_at = masked.updated_at.clone();
1637
1638 if masked.state_reason == StateReasonField::Rejected {
1647 report.warnings.push(format!(
1648 "issue #{number}: stateReason is not one of the governed values, record skipped"
1649 ));
1650 cursor_stalled = true;
1651 continue;
1652 }
1653
1654 let content = masked.body;
1655 let safe_title = masked.title;
1656 let mut properties = json!({
1657 "number": number,
1658 "title": safe_title,
1659 "author": masked.author_login,
1660 "created_at": masked.created_at,
1661 "closed_at": masked.closed_at,
1662 "labels": masked.labels,
1663 "project_id": project_id.to_string(),
1664 });
1665 if let StateReasonField::Valid(reason) = masked.state_reason {
1666 properties["state_reason"] = json!(reason);
1667 }
1668 let name = refs::truncate_chars(&format!("#{number} {safe_title}"), NAME_MAX_CHARS);
1669
1670 budget.try_consume();
1671 let result = match registry
1672 .dispatch(
1673 "create",
1674 json!({
1675 "kind": "issue",
1676 "name": name,
1677 "content": content,
1678 "properties": properties,
1679 "annotates": [project_id.to_string()],
1680 }),
1681 )
1682 .await
1683 {
1684 Ok(v) => v,
1685 Err(e) => {
1686 report.warnings.push(format!("create issue #{number}: {e}"));
1687 cursor_stalled = true;
1688 continue;
1689 }
1690 };
1691 if let Some(id) = result
1692 .get("id")
1693 .and_then(|v| v.as_str())
1694 .and_then(|s| Uuid::parse_str(s).ok())
1695 {
1696 new_records.push(NewRecordForRef {
1697 id,
1698 text: content.clone(),
1699 });
1700 }
1701
1702 report.issues_ingested += 1;
1703 if !cursor_stalled {
1704 if let Some(u) = &updated_at {
1705 if max_updated
1706 .as_deref()
1707 .map(|m| u.as_str() > m)
1708 .unwrap_or(true)
1709 {
1710 max_updated = Some(u.clone());
1711 }
1712 }
1713 }
1714 }
1715
1716 match decide_page_outcome(
1717 page_len,
1718 floor.as_deref(),
1719 last_updated_at.as_deref(),
1720 budget.exhausted(),
1721 ) {
1722 PageOutcome::WindowComplete => break 'paging,
1723 PageOutcome::StopBudgetExhausted | PageOutcome::StopFloorStalled => {
1724 window_complete = false;
1725 break 'paging;
1726 }
1727 PageOutcome::Continue(next_floor) => floor = Some(next_floor),
1728 }
1729 }
1730
1731 if !window_complete {
1732 report.done = false;
1733 }
1734
1735 if let Some(cursor) = max_updated {
1736 write_cursor(runtime, project_id, "issues", &cursor).await?;
1737 }
1738 Ok(())
1739}
1740
1741#[cfg(test)]
1742mod paging_tests {
1743 use super::*;
1744
1745 #[test]
1746 fn search_query_omits_updated_qualifier_with_no_floor() {
1747 assert_eq!(search_query(None), "sort:updated-asc");
1748 }
1749
1750 #[test]
1751 fn search_query_includes_inclusive_updated_floor() {
1752 assert_eq!(
1753 search_query(Some("2024-01-01T00:00:00Z")),
1754 "sort:updated-asc updated:>=2024-01-01T00:00:00Z"
1755 );
1756 }
1757
1758 #[test]
1759 fn short_page_proves_window_complete_regardless_of_budget() {
1760 let outcome = decide_page_outcome(42, None, Some("2024-01-01T00:00:00Z"), false);
1761 assert_eq!(outcome, PageOutcome::WindowComplete);
1762 assert!(page_outcome_proves_window_complete(outcome));
1763
1764 let outcome = decide_page_outcome(0, None, None, true);
1768 assert_eq!(outcome, PageOutcome::WindowComplete);
1769 }
1770
1771 #[test]
1778 fn full_page_with_stalled_floor_is_not_window_complete_even_with_budget_left() {
1779 let outcome = decide_page_outcome(PAGE_LIMIT, Some("X"), Some("X"), false);
1780 assert_eq!(outcome, PageOutcome::StopFloorStalled);
1781 assert!(!page_outcome_proves_window_complete(outcome));
1782 }
1783
1784 #[test]
1785 fn full_page_with_advancing_floor_and_budget_left_continues() {
1786 let outcome = decide_page_outcome(PAGE_LIMIT, Some("A"), Some("B"), false);
1787 assert_eq!(outcome, PageOutcome::Continue("B".to_string()));
1788 assert!(!page_outcome_proves_window_complete(outcome));
1789 }
1790
1791 #[test]
1792 fn full_page_with_exhausted_budget_stops_without_proving_completeness() {
1793 let outcome = decide_page_outcome(PAGE_LIMIT, Some("A"), Some("B"), true);
1794 assert_eq!(outcome, PageOutcome::StopBudgetExhausted);
1795 assert!(!page_outcome_proves_window_complete(outcome));
1796 }
1797
1798 #[test]
1799 fn full_page_with_no_updated_at_stalls_rather_than_looping_forever() {
1800 let outcome = decide_page_outcome(PAGE_LIMIT, Some("A"), None, false);
1801 assert_eq!(outcome, PageOutcome::StopFloorStalled);
1802 }
1803}
1804
1805#[cfg(test)]
1811mod recovery_classifier_tests {
1812 use super::*;
1813
1814 fn err(phase: GitLogPhase, stderr: &str) -> GitLogError {
1815 GitLogError {
1816 phase,
1817 stderr: stderr.to_string(),
1818 }
1819 }
1820
1821 const REAL_WORLD_MESSAGE: &str = "fatal: deadbeefdeadbeefdeadbeefdeadbeefdeadbeef is in \
1822 the commit graph file, but not in the object database\nfatal: unable to parse commit: \
1823 deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\nfatal: could not fetch from promisor remote";
1824
1825 #[test]
1826 fn classifies_real_world_missing_promisor_object_message_on_either_phase() {
1827 assert!(err(GitLogPhase::TouchedFiles, REAL_WORLD_MESSAGE).is_missing_promisor_object());
1828 assert!(err(GitLogPhase::Metadata, REAL_WORLD_MESSAGE).is_missing_promisor_object());
1829 }
1830
1831 #[test]
1832 fn classifies_missing_object_wording_case_insensitively() {
1833 assert!(err(
1834 GitLogPhase::TouchedFiles,
1835 "FATAL: MISSING OBJECT abc123; PROMISOR remote unavailable"
1836 )
1837 .is_missing_promisor_object());
1838 }
1839
1840 #[test]
1841 fn does_not_classify_bad_object_without_promisor() {
1842 assert!(!err(GitLogPhase::Metadata, "fatal: bad object HEAD").is_missing_promisor_object());
1843 }
1844
1845 #[test]
1846 fn does_not_classify_auth_or_network_failures() {
1847 assert!(!err(
1848 GitLogPhase::Metadata,
1849 "fatal: Authentication failed for 'https://example.com/org/repo.git/'"
1850 )
1851 .is_missing_promisor_object());
1852 assert!(!err(
1853 GitLogPhase::TouchedFiles,
1854 "fatal: unable to access 'https://example.com/org/repo.git/': Could not resolve host"
1855 )
1856 .is_missing_promisor_object());
1857 }
1858
1859 #[test]
1860 fn does_not_classify_promisor_mention_without_missing_object_wording() {
1861 assert!(!err(
1865 GitLogPhase::Metadata,
1866 "fatal: promisor remote configured but unreachable"
1867 )
1868 .is_missing_promisor_object());
1869 }
1870
1871 #[test]
1881 fn recover_commit_snapshot_returns_no_warning_when_healthy() {
1882 let _env = crate::cache::ENV_MUTEX.blocking_lock();
1883 let dir = tempfile::tempdir().expect("tempdir");
1884 init_repo_with_commit(dir.path());
1885 let mut recover_calls = 0;
1886 let (snapshot, warning) = recover_commit_snapshot(dir.path(), None, |_repo, _err| {
1887 recover_calls += 1;
1888 Ok(None)
1889 })
1890 .expect("healthy repo loads");
1891 assert_eq!(snapshot.commits.len(), 1);
1892 assert_eq!(warning, None);
1893 assert_eq!(recover_calls, 0);
1894 }
1895
1896 #[test]
1903 fn recover_commit_snapshot_never_calls_recover_for_unclassified_failures() {
1904 let _env = crate::cache::ENV_MUTEX.blocking_lock();
1905 let dir = tempfile::tempdir().expect("tempdir");
1906 let mut recover_calls = 0;
1909 let result = recover_commit_snapshot(dir.path(), None, |_repo, _err| {
1910 recover_calls += 1;
1911 Ok(Some(RecoveredRepo {
1912 repo: dir.path().to_path_buf(),
1913 strategy: CacheRepairStrategy::Refetch,
1914 }))
1915 });
1916 assert!(result.is_err(), "a non-repo path must fail to load");
1917 assert_eq!(
1918 recover_calls, 0,
1919 "an unclassified failure must never invoke recover"
1920 );
1921 }
1922
1923 fn init_repo_with_commit(repo: &Path) {
1924 let run = |args: &[&str]| {
1925 let out = std::process::Command::new("git")
1926 .arg("-C")
1927 .arg(repo)
1928 .args(args)
1929 .output()
1930 .expect("spawn git");
1931 assert!(
1932 out.status.success(),
1933 "git {args:?} failed: {}",
1934 String::from_utf8_lossy(&out.stderr)
1935 );
1936 };
1937 run(&["init", "-q", "-b", "main"]);
1938 run(&["config", "user.email", "test@example.com"]);
1939 run(&["config", "user.name", "Test User"]);
1940 std::fs::write(repo.join("a.txt"), b"hello").unwrap();
1941 run(&["add", "a.txt"]);
1942 run(&["commit", "-q", "-m", "initial"]);
1943 }
1944}
1945
1946#[cfg(test)]
1947mod truncation_tests {
1948 use super::*;
1949
1950 #[test]
1951 fn under_cap_content_is_not_truncated() {
1952 let content = "a".repeat(MAX_COMMIT_EMBED_BYTES - 1);
1953 assert_eq!(truncated_embedding_head(&content), None);
1954 }
1955
1956 #[test]
1957 fn exactly_at_cap_content_is_not_truncated() {
1958 let content = "a".repeat(MAX_COMMIT_EMBED_BYTES);
1959 assert_eq!(truncated_embedding_head(&content), None);
1960 }
1961
1962 #[test]
1963 fn over_cap_content_is_truncated_to_exactly_the_cap() {
1964 let content = "a".repeat(MAX_COMMIT_EMBED_BYTES + 1);
1965 let head = truncated_embedding_head(&content).expect("over cap must truncate");
1966 assert_eq!(head.len(), MAX_COMMIT_EMBED_BYTES);
1967 assert!(content.starts_with(head));
1968 }
1969
1970 #[test]
1974 fn multibyte_scalar_straddling_cap_rolls_back_to_char_boundary() {
1975 let mut content = "a".repeat(MAX_COMMIT_EMBED_BYTES - 1);
1978 content.push('€'); content.push_str("tail-sentinel");
1980
1981 let head = truncated_embedding_head(&content).expect("over cap must truncate");
1982 assert!(head.len() <= MAX_COMMIT_EMBED_BYTES);
1983 assert!(content.is_char_boundary(head.len()));
1984 assert!(std::str::from_utf8(head.as_bytes()).is_ok());
1985 assert!(content.starts_with(head));
1986 assert!(
1987 !head.contains("tail-sentinel"),
1988 "head must not include text past the cap"
1989 );
1990 }
1991}
1992
1993#[cfg(test)]
2000mod compact_prefix_resolver_tests {
2001 use super::*;
2002 use khive_runtime::Namespace;
2003
2004 #[tokio::test]
2005 async fn resolve_project_id_rejects_like_wildcard_input() {
2006 let rt = KhiveRuntime::memory().unwrap();
2007 let token = rt.authorize(Namespace::local()).unwrap();
2008 let project = rt
2009 .create_entity(
2010 &token,
2011 "project",
2012 None,
2013 "WildcardIngestTest",
2014 None,
2015 None,
2016 vec![],
2017 )
2018 .await
2019 .unwrap();
2020 let compact = project.id.simple().to_string();
2021 let wildcard_input = format!("{}%", &compact[..8]);
2022
2023 let resolved = resolve_project_id(&rt, &wildcard_input).await.unwrap();
2024 assert_eq!(
2025 resolved, None,
2026 "a %-bearing project argument must not resolve via a wildcard LIKE scan"
2027 );
2028 }
2029
2030 #[tokio::test]
2031 async fn resolve_id_resolves_compact_prefix_over_8_chars() {
2032 let rt = KhiveRuntime::memory().unwrap();
2033 let token = rt.authorize(Namespace::local()).unwrap();
2034 let project = rt
2035 .create_entity(
2036 &token,
2037 "project",
2038 None,
2039 "CompactIngestTest",
2040 None,
2041 None,
2042 vec![],
2043 )
2044 .await
2045 .unwrap();
2046 let compact = project.id.simple().to_string();
2047
2048 let resolved = resolve_id(&rt, &token, &compact[..16]).await.unwrap();
2049 assert_eq!(resolved, Some(project.id));
2050 }
2051}
2052
2053#[cfg(test)]
2058mod find_document_for_path_tests {
2059 use super::*;
2060 use khive_runtime::Namespace;
2061
2062 async fn create_document(rt: &KhiveRuntime, token: &NamespaceToken, source_uri: &str) -> Uuid {
2063 rt.create_entity(
2064 token,
2065 "document",
2066 None,
2067 source_uri,
2068 None,
2069 Some(json!({ "source_uri": source_uri })),
2070 vec![],
2071 )
2072 .await
2073 .unwrap()
2074 .id
2075 }
2076
2077 #[tokio::test]
2078 async fn path_with_like_wildcards_resolves_only_itself() {
2079 let rt = KhiveRuntime::memory().unwrap();
2080 let token = rt.authorize(Namespace::local()).unwrap();
2081
2082 let path = "src/100%_done.rs";
2090 let decoy_source_uri = "prefix/src/100Qdone.rs";
2091 create_document(&rt, &token, decoy_source_uri).await;
2092
2093 let resolved = find_document_for_path(&rt, &token, path).await.unwrap();
2094 assert_eq!(
2095 resolved, None,
2096 "a % or _ in the path must be matched literally, not as a LIKE wildcard"
2097 );
2098 }
2099
2100 #[tokio::test]
2101 async fn exact_match_wins_over_wildcard_broadened_candidate() {
2102 let rt = KhiveRuntime::memory().unwrap();
2103 let token = rt.authorize(Namespace::local()).unwrap();
2104
2105 let path = "crates/khive-pack-git/src/ingest.rs";
2106 let broadened_suffix_path = "other/crates/khive-pack-git/src/ingest.rs";
2107 create_document(&rt, &token, broadened_suffix_path).await;
2110 let exact_id = create_document(&rt, &token, path).await;
2111
2112 let resolved = find_document_for_path(&rt, &token, path).await.unwrap();
2113 assert_eq!(
2114 resolved,
2115 Some(exact_id),
2116 "an exact source_uri match must always win over a suffix-LIKE candidate"
2117 );
2118 }
2119
2120 #[tokio::test]
2127 async fn single_query_snapshot_prefers_exact_over_broadened() {
2128 let rt = KhiveRuntime::memory().unwrap();
2129 let token = rt.authorize(Namespace::local()).unwrap();
2130
2131 let path = "crates/khive-pack-git/src/toctou.rs";
2132 let broadened_suffix_path = "other/crates/khive-pack-git/src/toctou.rs";
2133 let exact_id = create_document(&rt, &token, path).await;
2134 create_document(&rt, &token, broadened_suffix_path).await;
2135
2136 let resolved = find_document_for_path(&rt, &token, path).await.unwrap();
2137 assert_eq!(
2138 resolved,
2139 Some(exact_id),
2140 "a single query covering both exact and broadened candidates \
2141 must still rank the exact match first, regardless of insertion order"
2142 );
2143 }
2144}