1use chrono::{DateTime, Utc};
7use surrealdb::Surreal;
8
9use super::confidence;
10use super::crud;
11use super::error::GraphError;
12use super::store::Db;
13use super::types::{Entity, Relationship};
14
15#[derive(Debug, Clone)]
17pub struct GcConfig {
18 pub stale_days: u64,
20 pub stale_confidence: f64,
22 pub dead_confidence: f64,
24 pub dead_min_age_days: u64,
26 pub dry_run: bool,
28 pub protect_pipeline: bool,
30}
31
32impl Default for GcConfig {
33 fn default() -> Self {
34 Self {
35 stale_days: 30,
36 stale_confidence: 0.5,
37 dead_confidence: 0.2,
38 dead_min_age_days: 14,
39 dry_run: true,
40 protect_pipeline: true,
41 }
42 }
43}
44
45#[derive(Debug, Clone)]
47pub struct GcAction {
48 pub target_id: String,
49 pub target_name: String,
50 pub kind: GcActionKind,
51 pub reason: String,
52}
53
54#[derive(Debug, Clone, PartialEq)]
56pub enum GcActionKind {
57 StaleRelationship,
58 DeadRelationship,
59 OrphanedEntity,
60}
61
62impl std::fmt::Display for GcActionKind {
63 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
64 match self {
65 Self::StaleRelationship => write!(f, "stale_relationship"),
66 Self::DeadRelationship => write!(f, "dead_relationship"),
67 Self::OrphanedEntity => write!(f, "orphaned_entity"),
68 }
69 }
70}
71
72#[derive(Debug, Clone, Default)]
74pub struct GcReport {
75 pub entities_scanned: u64,
76 pub relationships_scanned: u64,
77 pub stale_relationships: u64,
78 pub dead_relationships: u64,
79 pub orphaned_entities: u64,
80 pub total_removed: u64,
81 pub dry_run: bool,
82 pub actions: Vec<GcAction>,
83 pub errors: Vec<String>,
84}
85
86pub async fn run_gc(db: &Surreal<Db>, config: &GcConfig) -> Result<GcReport, GraphError> {
88 let now = Utc::now();
89 let mut report = GcReport {
90 dry_run: config.dry_run,
91 ..Default::default()
92 };
93
94 let all_rels = crud::list_all_relationships(db).await?;
96 let all_entities = crud::list_entities(db, None).await?;
97 report.relationships_scanned = all_rels.len() as u64;
98 report.entities_scanned = all_entities.len() as u64;
99
100 let stale_ids = phase_stale_relationships(&all_rels, config, &now, &mut report);
102
103 let dead_ids = phase_dead_relationships(&all_rels, config, &now, &stale_ids, &mut report);
105
106 let mut rel_ids_to_delete: Vec<String> = Vec::new();
108 rel_ids_to_delete.extend(stale_ids);
109 rel_ids_to_delete.extend(dead_ids);
110
111 let orphan_ids =
113 phase_orphaned_entities(db, &all_entities, config, &rel_ids_to_delete, &mut report).await?;
114
115 if !config.dry_run {
117 for rel_id in &rel_ids_to_delete {
118 if let Err(e) = crud::delete_relationship(db, rel_id).await {
119 report
120 .errors
121 .push(format!("Failed to delete relationship {rel_id}: {e}"));
122 } else {
123 report.total_removed += 1;
124 }
125 }
126
127 for entity_id in &orphan_ids {
128 if let Err(e) = crud::delete_entity(db, entity_id).await {
129 report
130 .errors
131 .push(format!("Failed to delete entity {entity_id}: {e}"));
132 } else {
133 report.total_removed += 1;
134 }
135 }
136 } else {
137 report.total_removed = rel_ids_to_delete.len() as u64 + orphan_ids.len() as u64;
138 }
139
140 Ok(report)
141}
142
143fn phase_stale_relationships(
147 rels: &[Relationship],
148 config: &GcConfig,
149 now: &DateTime<Utc>,
150 report: &mut GcReport,
151) -> Vec<String> {
152 let mut stale_ids = Vec::new();
153
154 for rel in rels {
155 if rel.valid_until.is_some() {
157 continue;
158 }
159
160 let effective = confidence::effective_confidence(
162 rel.confidence,
163 rel.last_reinforced.as_ref(),
164 &rel.valid_from,
165 now,
166 );
167
168 if effective >= config.stale_confidence {
170 continue;
171 }
172
173 let age_days = match parse_datetime(&rel.valid_from) {
175 Some(dt) => (*now - dt).num_days(),
176 None => continue,
177 };
178
179 if age_days < config.stale_days as i64 {
180 continue;
181 }
182
183 let id = rel.id_string();
184 let description = rel.description.as_deref().unwrap_or("(no description)");
185 report.actions.push(GcAction {
186 target_id: id.clone(),
187 target_name: format!(
188 "{} --[{}]--> {}",
189 value_to_short_id(&rel.from_id),
190 rel.rel_type,
191 value_to_short_id(&rel.to_id)
192 ),
193 kind: GcActionKind::StaleRelationship,
194 reason: format!(
195 "effective_confidence {:.2} (stored {:.2}) < {:.2}, age {} days > {}, desc: {}",
196 effective,
197 rel.confidence,
198 config.stale_confidence,
199 age_days,
200 config.stale_days,
201 description
202 ),
203 });
204 stale_ids.push(id);
205 report.stale_relationships += 1;
206 }
207
208 stale_ids
209}
210
211fn phase_dead_relationships(
214 rels: &[Relationship],
215 config: &GcConfig,
216 now: &DateTime<Utc>,
217 already_caught: &[String],
218 report: &mut GcReport,
219) -> Vec<String> {
220 let mut dead_ids = Vec::new();
221
222 for rel in rels {
223 let id = rel.id_string();
224
225 if already_caught.contains(&id) {
227 continue;
228 }
229
230 let effective = confidence::effective_confidence(
232 rel.confidence,
233 rel.last_reinforced.as_ref(),
234 &rel.valid_from,
235 now,
236 );
237
238 if effective >= config.dead_confidence {
240 continue;
241 }
242
243 let age_days = match parse_datetime(&rel.valid_from) {
245 Some(dt) => (*now - dt).num_days(),
246 None => continue,
247 };
248
249 if age_days < config.dead_min_age_days as i64 {
250 continue;
251 }
252
253 let description = rel.description.as_deref().unwrap_or("(no description)");
254 report.actions.push(GcAction {
255 target_id: id.clone(),
256 target_name: format!(
257 "{} --[{}]--> {}",
258 value_to_short_id(&rel.from_id),
259 rel.rel_type,
260 value_to_short_id(&rel.to_id)
261 ),
262 kind: GcActionKind::DeadRelationship,
263 reason: format!(
264 "effective_confidence {:.2} (stored {:.2}) < {:.2}, age {} days > {}, desc: {}",
265 effective,
266 rel.confidence,
267 config.dead_confidence,
268 age_days,
269 config.dead_min_age_days,
270 description
271 ),
272 });
273 dead_ids.push(id);
274 report.dead_relationships += 1;
275 }
276
277 dead_ids
278}
279
280async fn phase_orphaned_entities(
283 db: &Surreal<Db>,
284 entities: &[Entity],
285 config: &GcConfig,
286 pending_rel_deletions: &[String],
287 report: &mut GcReport,
288) -> Result<Vec<String>, GraphError> {
289 let mut orphan_ids = Vec::new();
290
291 for entity in entities {
292 if entity.access_count > 0 {
294 continue;
295 }
296
297 if config.protect_pipeline && is_pipeline_entity(entity) {
299 continue;
300 }
301
302 let entity_id = entity.id_string();
304 let current_rels = crud::count_relationships(db, &entity_id).await?;
305
306 let rels_being_deleted =
309 count_pending_deletions_for_entity(db, &entity_id, pending_rel_deletions).await?;
310
311 let remaining = current_rels.saturating_sub(rels_being_deleted);
312
313 if remaining > 0 {
314 continue;
315 }
316
317 report.actions.push(GcAction {
318 target_id: entity_id.clone(),
319 target_name: format!("{} ({})", entity.name, entity.entity_type),
320 kind: GcActionKind::OrphanedEntity,
321 reason: format!(
322 "zero relationships after pruning, access_count={}",
323 entity.access_count
324 ),
325 });
326 orphan_ids.push(entity_id);
327 report.orphaned_entities += 1;
328 }
329
330 Ok(orphan_ids)
331}
332
333async fn count_pending_deletions_for_entity(
335 db: &Surreal<Db>,
336 entity_id: &str,
337 pending_deletions: &[String],
338) -> Result<u64, GraphError> {
339 if pending_deletions.is_empty() {
340 return Ok(0);
341 }
342
343 let mut response = db
345 .query(
346 r#"SELECT id FROM relates_to
347 WHERE in = type::record($id) OR out = type::record($id)"#,
348 )
349 .bind(("id", entity_id.to_string()))
350 .await?;
351
352 #[derive(serde::Deserialize)]
353 struct IdRow {
354 id: serde_json::Value,
355 }
356
357 let rows: Vec<IdRow> = super::deserialize_take(&mut response, 0)?;
358 let count = rows
359 .iter()
360 .filter(|r| {
361 let id_str = match &r.id {
362 serde_json::Value::String(s) => s.clone(),
363 other => other.to_string(),
364 };
365 pending_deletions.contains(&id_str)
366 })
367 .count();
368
369 Ok(count as u64)
370}
371
372fn is_pipeline_entity(entity: &Entity) -> bool {
374 if let Some(ref source) = entity.source {
376 if source.starts_with("pipeline:") {
377 return true;
378 }
379 }
380
381 if let Some(ref attrs) = entity.attributes {
383 if attrs.get("pipeline_stage").is_some() {
384 return true;
385 }
386 }
387
388 false
389}
390
391use super::util::parse_datetime;
392
393fn value_to_short_id(val: &serde_json::Value) -> String {
395 match val {
396 serde_json::Value::String(s) => s.split(':').next_back().unwrap_or(s).to_string(),
397 other => other.to_string(),
398 }
399}
400
401pub async fn stats_only(db: &Surreal<Db>) -> Result<GcStatsReport, GraphError> {
404 let now = Utc::now();
405 let all_rels = crud::list_all_relationships(db).await?;
406 let all_entities = crud::list_entities(db, None).await?;
407
408 let pipeline_entities = all_entities
409 .iter()
410 .filter(|e| is_pipeline_entity(e))
411 .count();
412
413 let zero_access_entities = all_entities.iter().filter(|e| e.access_count == 0).count();
414
415 let low_confidence_rels = all_rels
416 .iter()
417 .filter(|r| {
418 confidence::effective_confidence(
419 r.confidence,
420 r.last_reinforced.as_ref(),
421 &r.valid_from,
422 &now,
423 ) < 0.5
424 })
425 .count();
426
427 let very_low_confidence_rels = all_rels
428 .iter()
429 .filter(|r| {
430 confidence::effective_confidence(
431 r.confidence,
432 r.last_reinforced.as_ref(),
433 &r.valid_from,
434 &now,
435 ) < 0.2
436 })
437 .count();
438
439 let superseded_rels = all_rels.iter().filter(|r| r.valid_until.is_some()).count();
440
441 Ok(GcStatsReport {
442 total_entities: all_entities.len() as u64,
443 total_relationships: all_rels.len() as u64,
444 pipeline_entities: pipeline_entities as u64,
445 zero_access_entities: zero_access_entities as u64,
446 low_confidence_rels: low_confidence_rels as u64,
447 very_low_confidence_rels: very_low_confidence_rels as u64,
448 superseded_rels: superseded_rels as u64,
449 })
450}
451
452#[derive(Debug, Clone)]
454pub struct GcStatsReport {
455 pub total_entities: u64,
456 pub total_relationships: u64,
457 pub pipeline_entities: u64,
458 pub zero_access_entities: u64,
459 pub low_confidence_rels: u64,
461 pub very_low_confidence_rels: u64,
463 pub superseded_rels: u64,
464}
465
466#[cfg(test)]
467mod tests {
468 use super::*;
469
470 #[test]
471 fn test_gc_config_defaults() {
472 let config = GcConfig::default();
473 assert_eq!(config.stale_days, 30);
474 assert_eq!(config.stale_confidence, 0.5);
475 assert_eq!(config.dead_confidence, 0.2);
476 assert_eq!(config.dead_min_age_days, 14);
477 assert!(config.dry_run);
478 assert!(config.protect_pipeline);
479 }
480
481 #[test]
482 fn test_parse_datetime_iso() {
483 let val = serde_json::Value::String("2024-01-15T10:30:00Z".to_string());
484 let dt = parse_datetime(&val);
485 assert!(dt.is_some());
486 }
487
488 #[test]
489 fn test_parse_datetime_invalid() {
490 let val = serde_json::Value::String("not-a-date".to_string());
491 let dt = parse_datetime(&val);
492 assert!(dt.is_none());
493 }
494
495 #[test]
496 fn test_parse_datetime_non_string() {
497 let val = serde_json::Value::Number(serde_json::Number::from(12345));
498 let dt = parse_datetime(&val);
499 assert!(dt.is_none());
500 }
501
502 #[test]
503 fn test_value_to_short_id() {
504 let val = serde_json::Value::String("entity:abc123".to_string());
505 assert_eq!(value_to_short_id(&val), "abc123");
506 }
507
508 #[test]
509 fn test_value_to_short_id_no_colon() {
510 let val = serde_json::Value::String("abc123".to_string());
511 assert_eq!(value_to_short_id(&val), "abc123");
512 }
513
514 #[test]
515 fn test_is_pipeline_entity_by_source() {
516 let entity = Entity {
517 id: serde_json::Value::String("entity:test".to_string()),
518 name: "Test".to_string(),
519 entity_type: super::super::types::EntityType::Thread,
520 abstract_text: "test".to_string(),
521 overview: "test".to_string(),
522 content: None,
523 attributes: None,
524 embedding: None,
525 mutable: true,
526 access_count: 0,
527 utility_score: 0.5,
528 utility_updates: 0,
529 created_at: serde_json::Value::String("2024-01-01T00:00:00Z".to_string()),
530 updated_at: serde_json::Value::String("2024-01-01T00:00:00Z".to_string()),
531 source: Some("pipeline:learning".to_string()),
532 };
533 assert!(is_pipeline_entity(&entity));
534 }
535
536 #[test]
537 fn test_is_pipeline_entity_by_attributes() {
538 let entity = Entity {
539 id: serde_json::Value::String("entity:test".to_string()),
540 name: "Test".to_string(),
541 entity_type: super::super::types::EntityType::Concept,
542 abstract_text: "test".to_string(),
543 overview: "test".to_string(),
544 content: None,
545 attributes: Some(serde_json::json!({"pipeline_stage": "thoughts"})),
546 embedding: None,
547 mutable: true,
548 access_count: 0,
549 utility_score: 0.5,
550 utility_updates: 0,
551 created_at: serde_json::Value::String("2024-01-01T00:00:00Z".to_string()),
552 updated_at: serde_json::Value::String("2024-01-01T00:00:00Z".to_string()),
553 source: None,
554 };
555 assert!(is_pipeline_entity(&entity));
556 }
557
558 #[test]
559 fn test_is_not_pipeline_entity() {
560 let entity = Entity {
561 id: serde_json::Value::String("entity:test".to_string()),
562 name: "Test".to_string(),
563 entity_type: super::super::types::EntityType::Tool,
564 abstract_text: "test".to_string(),
565 overview: "test".to_string(),
566 content: None,
567 attributes: None,
568 embedding: None,
569 mutable: true,
570 access_count: 0,
571 utility_score: 0.5,
572 utility_updates: 0,
573 created_at: serde_json::Value::String("2024-01-01T00:00:00Z".to_string()),
574 updated_at: serde_json::Value::String("2024-01-01T00:00:00Z".to_string()),
575 source: Some("llm:ingest".to_string()),
576 };
577 assert!(!is_pipeline_entity(&entity));
578 }
579
580 #[test]
581 fn test_phase_stale_relationships() {
582 let now = Utc::now();
583 let old_date = (now - chrono::Duration::days(45)).to_rfc3339();
584
585 let rels = vec![Relationship {
586 id: serde_json::Value::String("relates_to:abc".to_string()),
587 from_id: serde_json::Value::String("entity:a".to_string()),
588 to_id: serde_json::Value::String("entity:b".to_string()),
589 rel_type: "CONNECTED_TO".to_string(),
590 description: Some("test rel".to_string()),
591 valid_from: serde_json::Value::String(old_date),
592 valid_until: None,
593 confidence: 0.3,
594 last_reinforced: None,
595 source: Some("ingest".to_string()),
596 }];
597
598 let config = GcConfig::default();
599 let mut report = GcReport::default();
600 let stale = phase_stale_relationships(&rels, &config, &now, &mut report);
601
602 assert_eq!(stale.len(), 1);
603 assert_eq!(report.stale_relationships, 1);
604 }
605
606 #[test]
607 fn test_phase_stale_skips_high_confidence() {
608 let now = Utc::now();
609 let old_date = (now - chrono::Duration::days(45)).to_rfc3339();
610
611 let rels = vec![Relationship {
612 id: serde_json::Value::String("relates_to:abc".to_string()),
613 from_id: serde_json::Value::String("entity:a".to_string()),
614 to_id: serde_json::Value::String("entity:b".to_string()),
615 rel_type: "CONNECTED_TO".to_string(),
616 description: None,
617 valid_from: serde_json::Value::String(old_date),
618 valid_until: None,
619 confidence: 0.8,
620 last_reinforced: None,
621 source: None,
622 }];
623
624 let config = GcConfig::default();
625 let mut report = GcReport::default();
626 let stale = phase_stale_relationships(&rels, &config, &now, &mut report);
627
628 assert!(stale.is_empty());
629 }
630
631 #[test]
632 fn test_phase_stale_skips_young() {
633 let now = Utc::now();
634 let recent_date = (now - chrono::Duration::days(5)).to_rfc3339();
635
636 let rels = vec![Relationship {
637 id: serde_json::Value::String("relates_to:abc".to_string()),
638 from_id: serde_json::Value::String("entity:a".to_string()),
639 to_id: serde_json::Value::String("entity:b".to_string()),
640 rel_type: "CONNECTED_TO".to_string(),
641 description: None,
642 valid_from: serde_json::Value::String(recent_date),
643 valid_until: None,
644 confidence: 0.3,
645 last_reinforced: None,
646 source: None,
647 }];
648
649 let config = GcConfig::default();
650 let mut report = GcReport::default();
651 let stale = phase_stale_relationships(&rels, &config, &now, &mut report);
652
653 assert!(stale.is_empty());
654 }
655
656 #[test]
657 fn test_phase_stale_skips_superseded() {
658 let now = Utc::now();
659 let old_date = (now - chrono::Duration::days(45)).to_rfc3339();
660
661 let rels = vec![Relationship {
662 id: serde_json::Value::String("relates_to:abc".to_string()),
663 from_id: serde_json::Value::String("entity:a".to_string()),
664 to_id: serde_json::Value::String("entity:b".to_string()),
665 rel_type: "CONNECTED_TO".to_string(),
666 description: None,
667 valid_from: serde_json::Value::String(old_date.clone()),
668 valid_until: Some(serde_json::Value::String(old_date)),
669 confidence: 0.3,
670 last_reinforced: None,
671 source: None,
672 }];
673
674 let config = GcConfig::default();
675 let mut report = GcReport::default();
676 let stale = phase_stale_relationships(&rels, &config, &now, &mut report);
677
678 assert!(stale.is_empty());
679 }
680
681 #[test]
682 fn test_phase_dead_relationships() {
683 let now = Utc::now();
684 let old_date = (now - chrono::Duration::days(20)).to_rfc3339();
685
686 let rels = vec![Relationship {
687 id: serde_json::Value::String("relates_to:dead1".to_string()),
688 from_id: serde_json::Value::String("entity:a".to_string()),
689 to_id: serde_json::Value::String("entity:b".to_string()),
690 rel_type: "CONNECTED_TO".to_string(),
691 description: None,
692 valid_from: serde_json::Value::String(old_date),
693 valid_until: None,
694 confidence: 0.1,
695 last_reinforced: None,
696 source: None,
697 }];
698
699 let config = GcConfig::default();
700 let mut report = GcReport::default();
701 let already_caught = vec![];
702 let dead = phase_dead_relationships(&rels, &config, &now, &already_caught, &mut report);
703
704 assert_eq!(dead.len(), 1);
705 assert_eq!(report.dead_relationships, 1);
706 }
707
708 #[test]
709 fn test_phase_dead_skips_already_caught() {
710 let now = Utc::now();
711 let old_date = (now - chrono::Duration::days(20)).to_rfc3339();
712
713 let rels = vec![Relationship {
714 id: serde_json::Value::String("relates_to:dead1".to_string()),
715 from_id: serde_json::Value::String("entity:a".to_string()),
716 to_id: serde_json::Value::String("entity:b".to_string()),
717 rel_type: "CONNECTED_TO".to_string(),
718 description: None,
719 valid_from: serde_json::Value::String(old_date),
720 valid_until: None,
721 confidence: 0.1,
722 last_reinforced: None,
723 source: None,
724 }];
725
726 let config = GcConfig::default();
727 let mut report = GcReport::default();
728 let already_caught = vec!["relates_to:dead1".to_string()];
729 let dead = phase_dead_relationships(&rels, &config, &now, &already_caught, &mut report);
730
731 assert!(dead.is_empty());
732 }
733
734 #[test]
735 fn test_gc_action_kind_display() {
736 assert_eq!(
737 GcActionKind::StaleRelationship.to_string(),
738 "stale_relationship"
739 );
740 assert_eq!(
741 GcActionKind::DeadRelationship.to_string(),
742 "dead_relationship"
743 );
744 assert_eq!(GcActionKind::OrphanedEntity.to_string(), "orphaned_entity");
745 }
746
747 #[test]
748 fn test_phase_stale_decay_makes_high_stored_confidence_stale() {
749 let now = Utc::now();
752 let old_date = (now - chrono::Duration::days(180)).to_rfc3339();
753
754 let rels = vec![Relationship {
755 id: serde_json::Value::String("relates_to:decayed".to_string()),
756 from_id: serde_json::Value::String("entity:a".to_string()),
757 to_id: serde_json::Value::String("entity:b".to_string()),
758 rel_type: "CONNECTED_TO".to_string(),
759 description: Some("decayed rel".to_string()),
760 valid_from: serde_json::Value::String(old_date),
761 valid_until: None,
762 confidence: 0.6, last_reinforced: None, source: None,
765 }];
766
767 let config = GcConfig::default();
768 let mut report = GcReport::default();
769 let stale = phase_stale_relationships(&rels, &config, &now, &mut report);
770
771 assert_eq!(
774 stale.len(),
775 1,
776 "decayed relationship should be caught as stale"
777 );
778 }
779
780 #[test]
781 fn test_phase_stale_reinforced_prevents_decay() {
782 let now = Utc::now();
784 let old_date = (now - chrono::Duration::days(180)).to_rfc3339();
785 let recent_reinforce = (now - chrono::Duration::days(5)).to_rfc3339();
786
787 let rels = vec![Relationship {
788 id: serde_json::Value::String("relates_to:reinforced".to_string()),
789 from_id: serde_json::Value::String("entity:a".to_string()),
790 to_id: serde_json::Value::String("entity:b".to_string()),
791 rel_type: "CONNECTED_TO".to_string(),
792 description: None,
793 valid_from: serde_json::Value::String(old_date),
794 valid_until: None,
795 confidence: 0.6,
796 last_reinforced: Some(serde_json::Value::String(recent_reinforce)),
797 source: None,
798 }];
799
800 let config = GcConfig::default();
801 let mut report = GcReport::default();
802 let stale = phase_stale_relationships(&rels, &config, &now, &mut report);
803
804 assert!(
806 stale.is_empty(),
807 "recently reinforced relationship should NOT be stale"
808 );
809 }
810}