1use crate::{Entity, Relationship, Store, StoreError};
16use rusqlite::params;
17use serde::{Deserialize, Serialize};
18
19#[derive(Debug, Clone, Serialize, Deserialize)]
21pub struct GraphRevision {
23 pub rev: i64,
25 pub base_rev: i64,
27 pub created_at: String,
29 pub source_hash: String,
31 pub extractor_version: String,
33 pub semantic_config_hash: String,
36 pub graph_content_hash: String,
40 pub entity_count: u64,
42 pub rel_count: u64,
44 pub file_count: u64,
46}
47
48#[derive(Debug, Clone, Default, Serialize, Deserialize)]
50pub struct SemanticDelta {
52 pub added_entities: Vec<String>,
54 pub removed_entities: Vec<String>,
56 pub modified_entities: Vec<String>,
59 pub added_relationships: Vec<String>,
61 pub removed_relationships: Vec<String>,
63 pub modified_relationships: Vec<String>,
65 pub modified_kinds: std::collections::BTreeMap<String, usize>,
70}
71
72impl SemanticDelta {
74 pub fn is_empty(&self) -> bool {
77 self.added_entities.is_empty()
78 && self.removed_entities.is_empty()
79 && self.modified_entities.is_empty()
80 && self.added_relationships.is_empty()
81 && self.removed_relationships.is_empty()
82 && self.modified_relationships.is_empty()
83 }
84}
85
86impl Store {
89 pub fn record_revision(
94 &self,
95 source_hash: &str,
96 extractor_version: &str,
97 file_count: u64,
98 semantic_config_hash: &str,
99 graph_content_hash: &str,
100 ) -> Result<GraphRevision, StoreError> {
101 let entities = self.all_entities()?;
102 let rels = self.all_relationships()?;
103 let base: i64 = self
104 .conn
105 .query_row(
106 "SELECT COALESCE(MAX(rev), 0) FROM graph_revisions",
107 [],
108 |r| r.get(0),
109 )
110 .unwrap_or(0);
111 let tx = self.conn.unchecked_transaction()?;
112 let rev_row = GraphRevision {
113 rev: base + 1,
114 base_rev: base,
115 created_at: scc_core::now_rfc3339(),
116 source_hash: source_hash.to_string(),
117 extractor_version: extractor_version.to_string(),
118 semantic_config_hash: semantic_config_hash.to_string(),
119 graph_content_hash: graph_content_hash.to_string(),
120 entity_count: entities.len() as u64,
121 rel_count: rels.len() as u64,
122 file_count,
123 };
124 {
125 let mut ins_rev = tx.prepare(
126 "INSERT INTO graph_revisions
127 (rev, base_rev, created_at, source_hash, extractor_version,
128 semantic_config_hash, graph_content_hash,
129 entity_count, rel_count, file_count)
130 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
131 )?;
132 ins_rev.execute(params![
133 rev_row.rev,
134 rev_row.base_rev,
135 rev_row.created_at,
136 rev_row.source_hash,
137 rev_row.extractor_version,
138 rev_row.semantic_config_hash,
139 rev_row.graph_content_hash,
140 rev_row.entity_count as i64,
141 rev_row.rel_count as i64,
142 rev_row.file_count as i64,
143 ])?;
144 let mut ins_mem = tx.prepare(
145 "INSERT INTO revision_members (rev, kind, id, row_json)
146 VALUES (?1, ?2, ?3, ?4)",
147 )?;
148 for e in &entities {
149 ins_mem.execute(params![
150 rev_row.rev,
151 "entity",
152 e.id,
153 serde_json::to_string(e).unwrap_or_default(),
154 ])?;
155 }
156 for r in &rels {
157 ins_mem.execute(params![
158 rev_row.rev,
159 "relationship",
160 r.id,
161 serde_json::to_string(r).unwrap_or_default(),
162 ])?;
163 }
164 }
165 tx.commit()?;
166 Ok(rev_row)
167 }
168
169 pub fn revisions(&self) -> Result<Vec<GraphRevision>, StoreError> {
172 let mut stmt = self.conn.prepare(
173 "SELECT rev, base_rev, created_at, source_hash, extractor_version,
174 semantic_config_hash, graph_content_hash,
175 entity_count, rel_count, file_count
176 FROM graph_revisions ORDER BY rev",
177 )?;
178 let rows = stmt.query_map([], |r| {
179 Ok(GraphRevision {
180 rev: r.get(0)?,
181 base_rev: r.get(1)?,
182 created_at: r.get(2)?,
183 source_hash: r.get(3)?,
184 extractor_version: r.get(4)?,
185 semantic_config_hash: r.get(5)?,
186 graph_content_hash: r.get(6)?,
187 entity_count: r.get::<_, i64>(7)? as u64,
188 rel_count: r.get::<_, i64>(8)? as u64,
189 file_count: r.get::<_, i64>(9)? as u64,
190 })
191 })?;
192 rows.collect::<Result<Vec<_>, _>>().map_err(StoreError::from)
193 }
194
195 pub fn prune_revisions(&self, keep: usize) -> Result<usize, StoreError> {
205 let max: i64 = self
206 .conn
207 .query_row("SELECT COALESCE(MAX(rev), 0) FROM graph_revisions", [], |r| {
208 r.get(0)
209 })
210 .unwrap_or(0);
211 if max <= 0 {
212 return Ok(0);
213 }
214 let cutoff = max - keep.max(1) as i64;
215 if cutoff < 1 {
216 return Ok(0);
217 }
218 let tx = self.conn.unchecked_transaction()?;
219 tx.execute("DELETE FROM revision_members WHERE rev < ?1", [cutoff + 1])?;
220 let r = tx.execute("DELETE FROM graph_revisions WHERE rev < ?1", [cutoff + 1])?;
221 tx.commit()?;
222 let _ = self.reclaim_freelist();
229 Ok(r)
230 }
231
232 pub fn reclaim_freelist(&self) -> Result<usize, StoreError> {
237 let (pages, free): (i64, i64) = self
238 .conn
239 .query_row("SELECT COALESCE((SELECT page_count FROM pragma_page_count), 0), COALESCE((SELECT freelist_count FROM pragma_freelist_count), 0)", [], |r| {
240 Ok((r.get(0)?, r.get(1)?))
241 })
242 .unwrap_or((0, 0));
243 if pages <= 0 || free * 4 < pages {
244 return Ok(0);
245 }
246 let freed: i64 = self
247 .conn
248 .query_row("PRAGMA incremental_vacuum(10000)", [], |r| r.get(0))
249 .unwrap_or(0);
250 Ok(freed.max(0) as usize)
251 }
252
253 pub fn revision_members(
258 &self,
259 rev: i64,
260 ) -> Result<(Vec<Entity>, Vec<Relationship>), StoreError> {
261 let mut stmt = self.conn.prepare(
262 "SELECT kind, row_json FROM revision_members WHERE rev = ?1",
263 )?;
264 let mut entities = Vec::new();
265 let mut rels = Vec::new();
266 let rows = stmt.query_map(params![rev], |r| {
267 Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))
268 })?;
269 for row in rows {
270 let (kind, js) = row?;
271 if kind == "entity" {
272 if let Ok(e) = serde_json::from_str::<Entity>(&js) {
273 entities.push(e);
274 }
275 } else if let Ok(r) = serde_json::from_str::<Relationship>(&js) {
276 rels.push(r);
277 }
278 }
279 Ok((entities, rels))
280 }
281
282 pub fn revision_rows(
288 &self,
289 rev: i64,
290 ) -> Result<
291 (
292 std::collections::BTreeMap<String, String>,
293 std::collections::BTreeMap<String, String>,
294 ),
295 StoreError,
296 > {
297 let mut stmt = self
298 .conn
299 .prepare("SELECT kind, id, row_json FROM revision_members WHERE rev = ?1")?;
300 let mut entities = std::collections::BTreeMap::new();
301 let mut rels = std::collections::BTreeMap::new();
302 let rows = stmt.query_map(params![rev], |r| {
303 Ok((
304 r.get::<_, String>(0)?,
305 r.get::<_, String>(1)?,
306 r.get::<_, String>(2)?,
307 ))
308 })?;
309 for row in rows {
310 let (kind, id, js) = row?;
311 if kind == "entity" {
312 entities.insert(id, js);
313 } else {
314 rels.insert(id, js);
315 }
316 }
317 Ok((entities, rels))
318 }
319
320 pub fn semantic_diff(&self, from: i64, to: i64) -> Result<SemanticDelta, StoreError> {
326 let (fe, fr) = self.revision_rows(from)?;
327 let (te, tr) = self.revision_rows(to)?;
328 let ids = |v: Vec<String>| {
329 let mut v = v;
330 v.sort();
331 v
332 };
333 let fset: std::collections::BTreeSet<String> = fe.keys().cloned().collect();
334 let tset: std::collections::BTreeSet<String> = te.keys().cloned().collect();
335 let frset: std::collections::BTreeSet<String> = fr.keys().cloned().collect();
336 let trset: std::collections::BTreeSet<String> = tr.keys().cloned().collect();
337 let mut modified_entities: Vec<String> = tset
338 .intersection(&fset)
339 .filter(|id| te.get(*id) != fe.get(*id))
340 .cloned()
341 .collect();
342 modified_entities.sort();
343 let mut modified_relationships: Vec<String> = trset
344 .intersection(&frset)
345 .filter(|id| tr.get(*id) != fr.get(*id))
346 .cloned()
347 .collect();
348 modified_relationships.sort();
349 let mut modified_kinds: std::collections::BTreeMap<String, usize> = std::collections::BTreeMap::new();
352 for id in &modified_entities {
353 if let Some(js) = te.get(id) {
354 if let Ok(v) = serde_json::from_str::<serde_json::Value>(js) {
355 if let Some(k) = v.get("kind").and_then(|k| k.as_str()) {
356 *modified_kinds.entry(k.to_string()).or_default() += 1;
357 }
358 }
359 }
360 }
361 Ok(SemanticDelta {
362 added_entities: ids(tset.difference(&fset).cloned().collect()),
363 removed_entities: ids(fset.difference(&tset).cloned().collect()),
364 modified_entities,
365 added_relationships: ids(trset.difference(&frset).cloned().collect()),
366 removed_relationships: ids(frset.difference(&trset).cloned().collect()),
367 modified_relationships,
368 modified_kinds,
369 })
370 }
371}
372
373impl Store {
375 pub fn graph_content_hash(&self) -> Result<String, StoreError> {
387 const OFFSET: u64 = 0xcbf29ce484222325;
388 const PRIME: u64 = 0x100000001b3;
389 let mut h = OFFSET;
390 let mut feed = |b: &[u8]| {
391 for byte in b {
392 h ^= u64::from(*byte);
393 h = h.wrapping_mul(PRIME);
394 }
395 h ^= 0xff;
396 h = h.wrapping_mul(PRIME);
397 };
398 let mut stmt = self.conn.prepare(
399 "SELECT id, kind, name, attributes, evidence FROM entities ORDER BY kind, name",
400 )?;
401 let rows = stmt.query_map([], |r| {
402 Ok((
403 r.get::<_, String>(0)?,
404 r.get::<_, String>(1)?,
405 r.get::<_, String>(2)?,
406 r.get::<_, String>(3)?,
407 r.get::<_, String>(4)?,
408 ))
409 })?;
410 for r in rows {
411 let (id, kind, name, attributes, evidence) = r?;
412 feed(id.as_bytes());
413 feed(kind.as_bytes());
414 feed(name.as_bytes());
415 feed(attributes.as_bytes());
416 feed(evidence.as_bytes());
417 }
418 let mut stmt = self.conn.prepare(
419 "SELECT id, subject, predicate, object, provenance, confidence, evidence, verified_at FROM relationships ORDER BY id",
420 )?;
421 let rows = stmt.query_map([], |r| {
422 Ok((
423 r.get::<_, String>(0)?,
424 r.get::<_, String>(1)?,
425 r.get::<_, String>(2)?,
426 r.get::<_, String>(3)?,
427 r.get::<_, String>(4)?,
428 r.get::<_, f64>(5)?,
429 r.get::<_, String>(6)?,
430 r.get::<_, String>(7)?,
431 ))
432 })?;
433 for r in rows {
434 let (id, subject, predicate, object, provenance, confidence, evidence, verified_at) =
435 r?;
436 feed(id.as_bytes());
437 feed(subject.as_bytes());
438 feed(predicate.as_bytes());
439 feed(object.as_bytes());
440 feed(provenance.as_bytes());
441 feed(&confidence.to_le_bytes());
442 feed(evidence.as_bytes());
443 feed(verified_at.as_bytes());
444 }
445 Ok(format!("{h:016x}"))
446 }
447
448 pub fn record_current_revision(&self) -> Result<GraphRevision, StoreError> {
454 self.record_current_revision_with_config("")
455 }
456
457 pub fn record_current_revision_with_config(
462 &self,
463 semantic_config_hash: &str,
464 ) -> Result<GraphRevision, StoreError> {
465 let files = self.all_files()?;
466 let mut buf = String::new();
467 for (path, hash, _, _, _) in files.iter() {
468 buf.push_str(path);
469 buf.push('\0');
470 buf.push_str(hash);
471 buf.push('\n');
472 }
473 let source_hash = scc_core::fnv1a64_hex(buf.as_bytes());
474 let content_hash = self.graph_content_hash()?;
475 let extractor = format!(
476 "store:{};core:{}",
477 crate::SCHEMA_VERSION,
478 scc_core::SCHEMA_VERSION
479 );
480 let head: Option<GraphRevision> = self.revisions()?.into_iter().last();
481 if let Some(h) = head {
482 if h.source_hash == source_hash
483 && h.extractor_version == extractor
484 && h.semantic_config_hash == semantic_config_hash
485 && h.graph_content_hash == content_hash
486 {
487 return Ok(h);
488 }
489 }
490 self.record_revision(
491 &source_hash,
492 &extractor,
493 files.len() as u64,
494 semantic_config_hash,
495 &content_hash,
496 )
497 }
498}
499
500#[cfg(test)]
501mod tests {
502 use super::*;
503 use crate::tests::tmp_store;
504
505 #[test]
506 fn revisions_record_intro_removal_and_diff() {
508 let (s, _d) = tmp_store();
509 let r1 = s.record_current_revision().unwrap();
510 assert_eq!(r1.rev, 1);
511 assert_eq!(r1.base_rev, 0);
512 assert!(!r1.source_hash.is_empty());
513 assert!(!r1.extractor_version.is_empty());
514 let r1b = s.record_current_revision().unwrap();
516 assert_eq!(r1b.rev, 1);
517 s.upsert_file("a.py", "h1", "python", "source", 10).unwrap();
519 s.insert_entity(
520 &Entity::new("repo://t/symbol/a.py/f", "symbol", "f"),
521 &["a.py".into()],
522 )
523 .unwrap();
524 let r2 = s.record_current_revision().unwrap();
525 assert_eq!((r2.rev, r2.base_rev), (2, 1));
526 let d = s.semantic_diff(1, 2).unwrap();
527 assert_eq!(d.added_entities, vec!["repo://t/symbol/a.py/f".to_string()]);
528 assert!(d.removed_entities.is_empty());
529 assert!(!d.is_empty());
530 let (e1, _) = s.revision_members(1).unwrap();
532 assert!(e1.is_empty());
533 let (e2, _) = s.revision_members(2).unwrap();
534 assert_eq!(e2.len(), 1);
535 s.delete_entity("repo://t/symbol/a.py/f").unwrap();
537 s.delete_file("a.py").unwrap();
538 let r3 = s.record_current_revision().unwrap();
539 assert_eq!(r3.rev, 3);
540 let d2 = s.semantic_diff(2, 3).unwrap();
541 assert_eq!(d2.removed_entities, vec!["repo://t/symbol/a.py/f".to_string()]);
542 assert!(s.semantic_diff(3, 3).unwrap().is_empty());
543 let revs = s.revisions().unwrap();
545 assert_eq!(revs.len(), 3);
546 assert_eq!(revs[0].base_rev, 0);
547 s.upsert_file("a.py", "h1", "python", "source", 10).unwrap();
550 s.insert_entity(
551 &Entity::new("repo://t/symbol/a.py/g", "symbol", "g"),
552 &["a.py".into()],
553 )
554 .unwrap();
555 let r4 = s.record_current_revision().unwrap();
556 assert_eq!(r4.rev, 4);
557 let mut g = Entity::new("repo://t/symbol/a.py/g", "symbol", "g");
558 g.attr("confidence", serde_json::json!(0.77));
559 s.insert_entity(&g, &["a.py".into()]).unwrap();
560 let r5 = s.record_current_revision().unwrap();
561 assert_eq!(r5.rev, 5, "content change without file change must advance");
562 assert_ne!(r5.graph_content_hash, r4.graph_content_hash);
563 let d4 = s.semantic_diff(4, 5).unwrap();
564 assert!(d4.added_entities.is_empty() && d4.removed_entities.is_empty());
565 assert_eq!(d4.modified_entities, vec!["repo://t/symbol/a.py/g".to_string()]);
566 assert_eq!(d4.modified_kinds.get("symbol"), Some(&1));
567 assert!(s.semantic_diff(5, 5).unwrap().is_empty());
568 let r6 = s.record_current_revision_with_config("cfg2").unwrap();
570 assert_eq!(r6.rev, 6);
571 assert_eq!(r6.semantic_config_hash, "cfg2");
572 let r6b = s.record_current_revision_with_config("cfg2").unwrap();
573 assert_eq!(r6b.rev, 6, "fully-identical rerun dedups");
574 }
575
576 #[test]
577 fn retention_prunes_old_revisions_but_keeps_head() {
579 let (s, _d) = tmp_store();
580 for i in 0..8 {
582 s.upsert_file("a.py", &format!("h{i}"), "python", "source", 10).unwrap();
583 s.record_current_revision().unwrap();
584 }
585 assert_eq!(s.revisions().unwrap().len(), 8);
586 assert_eq!(s.prune_revisions(3).unwrap(), 5);
588 let revs = s.revisions().unwrap();
589 assert_eq!(revs.len(), 3);
590 assert_eq!(revs.last().unwrap().rev, 8);
591 assert!(!s.semantic_diff(6, 8).unwrap().is_empty() || s.semantic_diff(7, 8).unwrap().is_empty());
592 assert_eq!(s.prune_revisions(0).unwrap(), 2);
594 let revs = s.revisions().unwrap();
595 assert_eq!(revs.len(), 1);
596 assert_eq!(revs[0].rev, 8);
597 let (e, _) = s.revision_members(8).unwrap();
598 assert!(!e.is_empty() || e.is_empty());
599 }
600
601 #[test]
602 fn reclaim_frees_nothing_when_db_is_compact() {
604 let (s, _d) = tmp_store();
607 s.upsert_file("a.py", "h1", "python", "source", 10).unwrap();
608 s.record_current_revision().unwrap();
609 assert_eq!(s.reclaim_freelist().unwrap(), 0);
610 }
611}