1use std::collections::{BTreeMap, BTreeSet};
35
36use kimetsu_core::KimetsuResult;
37use rusqlite::Connection;
38
39use crate::consolidate::parse_tags;
40
41#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
43pub struct EdgeProposal {
44 pub src_id: String,
45 pub dst_id: String,
46 pub edge_type: String,
47}
48
49pub const RELATES_TO: &str = "relates_to";
51
52pub const DEFAULT_MAX_FAN_OUT: usize = 8;
55
56const MIN_KEYWORD_LEN: usize = 5;
59
60const STOPWORDS: &[&str] = &[
63 "about", "above", "after", "again", "against", "always", "because", "before", "being", "below",
64 "between", "could", "default", "during", "every", "first", "found", "their", "there", "these",
65 "thing", "things", "those", "through", "under", "until", "using", "value", "where", "which",
66 "while", "would", "should", "while",
67];
68
69#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
73pub enum EntitySource {
74 Tag,
76 Term,
78}
79
80impl EntitySource {
81 pub fn as_str(self) -> &'static str {
82 match self {
83 EntitySource::Tag => "tag",
84 EntitySource::Term => "term",
85 }
86 }
87}
88
89pub fn extract_entities_with_source(text: &str) -> Vec<(String, EntitySource)> {
95 let mut sources: BTreeMap<String, EntitySource> = BTreeMap::new();
96 for tag in tag_entities(text) {
97 sources.insert(tag, EntitySource::Tag);
98 }
99 for term in term_entities(text) {
100 sources.entry(term).or_insert(EntitySource::Term);
101 }
102 sources.into_iter().collect()
103}
104
105fn tag_entities(text: &str) -> BTreeSet<String> {
107 let mut set = BTreeSet::new();
108 for t in parse_tags(text) {
112 for word in t.split_whitespace() {
113 let w = word.trim();
114 if w.len() >= 3 {
115 set.insert(w.to_string());
116 }
117 }
118 }
119 set
120}
121
122fn term_entities(text: &str) -> BTreeSet<String> {
124 let mut set = BTreeSet::new();
125 for raw in text.split(|c: char| !c.is_alphanumeric()) {
126 if raw.is_empty() {
127 continue;
128 }
129 let is_proper = raw.chars().next().is_some_and(|c| c.is_uppercase())
130 && raw.chars().skip(1).any(|c| c.is_lowercase());
131 let lower = raw.to_ascii_lowercase();
132 let proper_kept = is_proper && lower.len() >= 3;
135 let informative = lower.len() >= MIN_KEYWORD_LEN
136 && !STOPWORDS.contains(&lower.as_str())
137 && lower.chars().any(|c| c.is_alphabetic());
138 if proper_kept || informative {
139 set.insert(lower);
140 }
141 }
142 set
143}
144
145pub fn extract_entities(text: &str) -> Vec<String> {
155 let mut set = tag_entities(text);
156 set.extend(term_entities(text));
157 set.into_iter().collect()
158}
159
160fn load_active_memories(conn: &Connection) -> KimetsuResult<Vec<(String, String)>> {
163 let mut stmt = conn.prepare(
164 "SELECT memory_id, text
165 FROM memories
166 WHERE invalidated_at IS NULL AND superseded_by IS NULL
167 ORDER BY memory_id",
168 )?;
169 let rows = stmt
170 .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))?
171 .collect::<Result<Vec<_>, _>>()?;
172 Ok(rows)
173}
174
175pub fn build_relates_to_edges(
185 conn: &Connection,
186 max_fan_out: usize,
187) -> KimetsuResult<Vec<EdgeProposal>> {
188 let cap = if max_fan_out == 0 {
189 DEFAULT_MAX_FAN_OUT
190 } else {
191 max_fan_out
192 };
193 let memories = load_active_memories(conn)?;
194
195 let mut by_entity: BTreeMap<String, Vec<String>> = BTreeMap::new();
197 for (id, text) in &memories {
198 for entity in extract_entities(text) {
199 by_entity.entry(entity).or_default().push(id.clone());
200 }
201 }
202
203 let mut pairs: BTreeSet<(String, String)> = BTreeSet::new();
205 for ids in by_entity.values() {
206 if ids.len() < 2 || ids.len() > cap.max(2) * 4 {
209 continue;
210 }
211 for i in 0..ids.len() {
212 for j in (i + 1)..ids.len() {
213 let (a, b) = if ids[i] < ids[j] {
214 (ids[i].clone(), ids[j].clone())
215 } else if ids[i] > ids[j] {
216 (ids[j].clone(), ids[i].clone())
217 } else {
218 continue; };
220 pairs.insert((a, b));
221 }
222 }
223 }
224
225 let mut fan_out: BTreeMap<String, usize> = BTreeMap::new();
227 let mut proposals: Vec<EdgeProposal> = Vec::new();
228 for (a, b) in pairs {
229 let ca = fan_out.entry(a.clone()).or_insert(0);
230 if *ca >= cap {
231 continue;
232 }
233 *ca += 1;
234 proposals.push(EdgeProposal {
235 src_id: a,
236 dst_id: b,
237 edge_type: RELATES_TO.to_string(),
238 });
239 }
240 Ok(proposals)
241}
242
243pub const INCREMENTAL_MIN_SHARED_ENTITIES: usize = 2;
252
253pub fn project_entities(conn: &Connection, memory_id: &str, text: &str) -> KimetsuResult<usize> {
256 conn.execute(
257 "DELETE FROM memory_entities WHERE memory_id = ?1",
258 rusqlite::params![memory_id],
259 )?;
260 let entities = extract_entities_with_source(text);
261 let mut stmt = conn.prepare(
262 "INSERT OR REPLACE INTO memory_entities (memory_id, entity, source) VALUES (?1, ?2, ?3)",
263 )?;
264 for (entity, source) in &entities {
265 stmt.execute(rusqlite::params![memory_id, entity, source.as_str()])?;
266 }
267 Ok(entities.len())
268}
269
270pub fn forget_entities(conn: &Connection, memory_id: &str) -> KimetsuResult<()> {
273 conn.execute(
274 "DELETE FROM memory_entities WHERE memory_id = ?1",
275 rusqlite::params![memory_id],
276 )?;
277 Ok(())
278}
279
280pub fn incremental_edges_for_memory(
294 conn: &Connection,
295 memory_id: &str,
296 max_fan_out: usize,
297) -> KimetsuResult<Vec<EdgeProposal>> {
298 let cap = if max_fan_out == 0 {
299 DEFAULT_MAX_FAN_OUT
300 } else {
301 max_fan_out
302 };
303
304 let mut stmt = conn.prepare(
307 "SELECT other.memory_id, COUNT(*) AS shared
308 FROM memory_entities AS mine
309 JOIN memory_entities AS other
310 ON other.entity = mine.entity AND other.memory_id != mine.memory_id
311 JOIN memories AS m
312 ON m.memory_id = other.memory_id
313 WHERE mine.memory_id = ?1
314 AND m.invalidated_at IS NULL
315 AND m.superseded_by IS NULL
316 GROUP BY other.memory_id
317 HAVING shared >= ?2
318 ORDER BY shared DESC, other.memory_id ASC
319 LIMIT ?3",
320 )?;
321 let neighbours = stmt
322 .query_map(
323 rusqlite::params![
324 memory_id,
325 INCREMENTAL_MIN_SHARED_ENTITIES as i64,
326 cap as i64
327 ],
328 |row| row.get::<_, String>(0),
329 )?
330 .collect::<Result<Vec<_>, _>>()?;
331
332 Ok(neighbours
333 .into_iter()
334 .map(|other| {
335 let (src_id, dst_id) = if memory_id < other.as_str() {
336 (memory_id.to_string(), other)
337 } else {
338 (other, memory_id.to_string())
339 };
340 EdgeProposal {
341 src_id,
342 dst_id,
343 edge_type: RELATES_TO.to_string(),
344 }
345 })
346 .collect())
347}
348
349pub fn reproject_all_entities(conn: &Connection) -> KimetsuResult<usize> {
353 let memories = load_active_memories(conn)?;
354 let mut total = 0usize;
355 for (id, text) in &memories {
356 total += project_entities(conn, id, text)?;
357 }
358 Ok(total)
359}
360
361#[cfg(test)]
362mod tests {
363 use super::*;
364 use crate::projector::add_memory_edges;
365 use crate::schema;
366 use rusqlite::params;
367
368 fn make_conn() -> Connection {
369 let conn = Connection::open_in_memory().expect("open_in_memory");
370 schema::initialize(&conn).expect("schema::initialize");
371 conn
372 }
373
374 fn insert_active_memory(conn: &Connection, id: &str, text: &str) {
375 conn.execute(
376 "INSERT INTO memories
377 (memory_id, scope, kind, text, normalized_text, confidence, provenance_snapshot_json, created_at)
378 VALUES (?1, 'global_user', 'fact', ?2, ?2, 0.85, '{}', '2024-01-01T00:00:00Z')",
379 params![id, text],
380 )
381 .expect("insert memory");
382 }
383
384 #[test]
385 fn extract_entities_picks_tags_and_salient_terms() {
386 let ents = extract_entities("[tags: rust mutex] Holding a Mutex across an await deadlocks");
387 assert!(ents.contains(&"rust".to_string()));
389 assert!(ents.contains(&"mutex".to_string()));
390 assert!(ents.contains(&"deadlocks".to_string()));
392 assert!(!ents.contains(&"a".to_string()));
393 assert!(!ents.contains(&"an".to_string()));
394 }
395
396 #[test]
397 fn extract_entities_is_sorted_and_deduped() {
398 let ents = extract_entities("Docker docker DOCKER mount mount");
399 let mut sorted = ents.clone();
400 sorted.sort();
401 assert_eq!(ents, sorted, "entities must be returned sorted");
402 let set: BTreeSet<&String> = ents.iter().collect();
403 assert_eq!(set.len(), ents.len(), "no duplicates");
404 }
405
406 #[test]
407 fn build_edges_links_shared_entity_and_skips_unrelated() {
408 let conn = make_conn();
409 insert_active_memory(
411 &conn,
412 "a",
413 "[tags: deadlock] holding a mutex guard deadlock risk",
414 );
415 insert_active_memory(
416 &conn,
417 "b",
418 "the async runtime can deadlock under contention",
419 );
420 insert_active_memory(
421 &conn,
422 "c",
423 "the website landing page uses a teal gradient hero",
424 );
425
426 let edges = build_relates_to_edges(&conn, 0).expect("build");
427 assert_eq!(edges.len(), 1, "got {edges:?}");
429 assert_eq!(edges[0].src_id, "a");
430 assert_eq!(edges[0].dst_id, "b");
431 assert_eq!(edges[0].edge_type, RELATES_TO);
432 }
433
434 #[test]
435 fn build_edges_persist_roundtrip() {
436 let conn = make_conn();
437 insert_active_memory(&conn, "a", "windows docker named pipe mount rule");
438 insert_active_memory(&conn, "b", "docker mount breaks under a tcp host");
439
440 let edges = build_relates_to_edges(&conn, 0).expect("build");
441 assert!(!edges.is_empty());
442 let tuples: Vec<(String, String, String)> = edges
443 .iter()
444 .map(|e| (e.src_id.clone(), e.dst_id.clone(), e.edge_type.clone()))
445 .collect();
446 let written = add_memory_edges(&conn, &tuples).expect("persist");
447 assert_eq!(written, edges.len());
448
449 let n: i64 = conn
450 .query_row(
451 "SELECT COUNT(*) FROM memory_edges WHERE edge_type='relates_to'",
452 [],
453 |r| r.get(0),
454 )
455 .unwrap();
456 assert_eq!(n as usize, edges.len());
457 }
458
459 #[test]
462 fn entity_source_prefers_the_author_supplied_tag() {
463 let pairs = extract_entities_with_source("[tags: mutex] Holding a mutex across an await");
464 let mutex = pairs
465 .iter()
466 .find(|(e, _)| e == "mutex")
467 .expect("mutex must be extracted");
468 assert_eq!(
469 mutex.1,
470 EntitySource::Tag,
471 "an entity that is both tagged and mentioned is a tag: the author said it out loud"
472 );
473 let holding = pairs.iter().find(|(e, _)| e == "holding");
474 assert_eq!(holding.map(|(_, s)| *s), Some(EntitySource::Term));
475 }
476
477 #[test]
478 fn project_entities_replaces_rather_than_accumulates() {
479 let conn = make_conn();
480 insert_active_memory(&conn, "a", "[tags: sqlite] vacuum reclaims dead pages");
481 project_entities(&conn, "a", "[tags: sqlite] vacuum reclaims dead pages").expect("project");
482 let first: i64 = conn
483 .query_row(
484 "SELECT COUNT(*) FROM memory_entities WHERE memory_id='a'",
485 [],
486 |r| r.get(0),
487 )
488 .unwrap();
489 assert!(first > 0, "entities must land");
490
491 project_entities(&conn, "a", "[tags: sqlite]").expect("reproject");
493 let entities: Vec<String> = conn
494 .prepare("SELECT entity FROM memory_entities WHERE memory_id='a' ORDER BY entity")
495 .unwrap()
496 .query_map([], |r| r.get(0))
497 .unwrap()
498 .collect::<Result<_, _>>()
499 .unwrap();
500 assert_eq!(entities, vec!["sqlite".to_string()]);
501 }
502
503 #[test]
507 fn incremental_edges_link_a_new_memory_to_its_neighbours() {
508 let conn = make_conn();
509 for (id, text) in [
510 (
511 "a",
512 "[tags: sqlite wal] WAL mode needs a checkpoint before backup",
513 ),
514 (
515 "b",
516 "[tags: sqlite wal] Opening a WAL database read-only skips recovery",
517 ),
518 ("c", "[tags: rust] Prefer thiserror for library error types"),
519 ] {
520 insert_active_memory(&conn, id, text);
521 project_entities(&conn, id, text).expect("project");
522 }
523
524 let edges = incremental_edges_for_memory(&conn, "b", 0).expect("incremental");
525 let linked: Vec<&str> = edges
526 .iter()
527 .map(|e| {
528 if e.src_id == "b" {
529 e.dst_id.as_str()
530 } else {
531 e.src_id.as_str()
532 }
533 })
534 .collect();
535 assert_eq!(
536 linked,
537 vec!["a"],
538 "two shared entities (sqlite, wal) links a-b; one topic in common does not link c"
539 );
540 let edge = &edges[0];
541 assert!(edge.src_id < edge.dst_id, "edges are stored src < dst");
542 assert_eq!(edge.edge_type, RELATES_TO);
543 }
544
545 #[test]
548 fn incremental_edges_need_more_than_one_shared_entity() {
549 let conn = make_conn();
550 for (id, text) in [
551 ("a", "[tags: sqlite] checkpoint before backup"),
552 ("b", "[tags: sqlite] different subject entirely"),
553 ] {
554 insert_active_memory(&conn, id, text);
555 project_entities(&conn, id, text).expect("project");
556 }
557 let shared: i64 = conn
559 .query_row(
560 "SELECT COUNT(*) FROM memory_entities x JOIN memory_entities y
561 ON x.entity = y.entity AND x.memory_id='a' AND y.memory_id='b'",
562 [],
563 |r| r.get(0),
564 )
565 .unwrap();
566 assert_eq!(shared, 1, "fixture must share exactly one entity");
567
568 let edges = incremental_edges_for_memory(&conn, "b", 0).expect("incremental");
569 assert!(
570 edges.is_empty(),
571 "one shared entity is not enough: {edges:?}"
572 );
573 }
574
575 #[test]
576 fn incremental_edges_skip_inactive_neighbours() {
577 let conn = make_conn();
578 for (id, text) in [
579 ("a", "[tags: sqlite wal] WAL mode needs a checkpoint"),
580 ("b", "[tags: sqlite wal] WAL recovery on read-only open"),
581 ] {
582 insert_active_memory(&conn, id, text);
583 project_entities(&conn, id, text).expect("project");
584 }
585 conn.execute(
586 "UPDATE memories SET invalidated_at = '2024-06-01T00:00:00Z' WHERE memory_id = 'a'",
587 [],
588 )
589 .unwrap();
590 let edges = incremental_edges_for_memory(&conn, "b", 0).expect("incremental");
591 assert!(
592 edges.is_empty(),
593 "an invalidated memory is not a reachable destination: {edges:?}"
594 );
595 }
596
597 #[test]
598 fn incremental_edges_respect_the_fan_out_cap() {
599 let conn = make_conn();
600 for i in 0..10 {
601 let id = format!("m{i}");
602 let text = "[tags: sqlite wal] shared topic";
603 insert_active_memory(&conn, &id, text);
604 project_entities(&conn, &id, text).expect("project");
605 }
606 let edges = incremental_edges_for_memory(&conn, "m0", 3).expect("incremental");
607 assert_eq!(edges.len(), 3, "fan-out cap must bound the write path");
608 }
609
610 #[test]
611 fn reproject_all_entities_backfills_an_existing_corpus() {
612 let conn = make_conn();
613 insert_active_memory(&conn, "a", "[tags: sqlite wal] checkpoint before backup");
614 insert_active_memory(&conn, "b", "[tags: sqlite wal] recovery on read-only open");
615 let before: i64 = conn
617 .query_row("SELECT COUNT(*) FROM memory_entities", [], |r| r.get(0))
618 .unwrap();
619 assert_eq!(before, 0);
620
621 reproject_all_entities(&conn).expect("backfill");
622 let edges = incremental_edges_for_memory(&conn, "b", 0).expect("incremental");
623 assert_eq!(
624 edges.len(),
625 1,
626 "backfilled index must be usable immediately"
627 );
628 }
629
630 #[test]
634 fn recording_memories_links_them_without_a_manual_graph_build() {
635 use kimetsu_core::memory::{MemoryKind, MemoryScope};
636
637 let ts = std::time::SystemTime::now()
638 .duration_since(std::time::UNIX_EPOCH)
639 .map(|d| d.as_nanos())
640 .unwrap_or(0);
641 let dir = std::env::temp_dir().join(format!("kimetsu-graph-ingest-{ts}"));
642 std::fs::create_dir_all(&dir).expect("create tmp");
643 kimetsu_core::paths::git_init_boundary(&dir);
644
645 crate::user_brain::with_user_brain_disabled(|| {
646 crate::project::init_project(&dir, true).expect("init");
647 crate::project::add_memory(
648 &dir,
649 MemoryScope::Project,
650 MemoryKind::Convention,
651 "[tags: sqlite wal] Checkpoint the WAL before copying brain.db",
652 )
653 .expect("add first");
654 crate::project::add_memory(
655 &dir,
656 MemoryScope::Project,
657 MemoryKind::FailurePattern,
658 "[tags: sqlite wal] Opening a WAL database read-only skips recovery",
659 )
660 .expect("add second");
661
662 let paths = kimetsu_core::paths::ProjectPaths::discover(&dir).expect("paths");
663 let conn = Connection::open(&paths.brain_db).expect("open brain");
664 let relates: i64 = conn
665 .query_row(
666 "SELECT COUNT(*) FROM memory_edges WHERE edge_type = 'relates_to'",
667 [],
668 |r| r.get(0),
669 )
670 .expect("count edges");
671 assert!(
672 relates >= 1,
673 "the write path must link related memories; got {relates} relates_to edges"
674 );
675
676 let entities: i64 = conn
677 .query_row("SELECT COUNT(*) FROM memory_entities", [], |r| r.get(0))
678 .expect("count entities");
679 assert!(entities > 0, "entities must be projected on write");
680 });
681
682 std::fs::remove_dir_all(dir).ok();
683 }
684
685 #[test]
686 fn build_edges_excludes_superseded() {
687 let conn = make_conn();
688 insert_active_memory(&conn, "a", "shared topic alpha beta gamma");
689 insert_active_memory(&conn, "b", "shared topic alpha beta gamma too");
690 conn.execute(
692 "UPDATE memories SET superseded_by = 'a' WHERE memory_id = 'b'",
693 [],
694 )
695 .unwrap();
696 let edges = build_relates_to_edges(&conn, 0).expect("build");
697 assert!(edges.is_empty(), "superseded memory must not be linked");
698 }
699}