use anyhow::{Context, Result};
use redb::{Database, ReadableTable, ReadableTableMetadata, TableDefinition};
use std::path::Path;
use std::sync::atomic::{AtomicU64, Ordering};
use uuid::Uuid;
#[derive(Debug, Clone)]
pub struct MatchRecord {
pub child_id: Uuid,
pub parent_id: Uuid,
pub jaccard_similarity: f64,
pub size_difference: i32,
pub size_difference_pct: f64,
}
fn match_key(child_id: Uuid, parent_id: Uuid) -> [u8; 32] {
let mut key = [0u8; 32];
key[..16].copy_from_slice(child_id.as_bytes());
key[16..].copy_from_slice(parent_id.as_bytes());
key
}
fn child_id_from_match_key(key: &[u8]) -> Result<Uuid> {
match key.len() {
16 | 32 => Ok(Uuid::from_slice(&key[..16])?),
len => anyhow::bail!("Invalid match key length: {}", len),
}
}
const MATCHES_TABLE: TableDefinition<&[u8], &[u8]> = TableDefinition::new("matches");
const ADJACENCY_TABLE: TableDefinition<&[u8], &[u8]> = TableDefinition::new("adjacency");
const ADJACENCY_BUILT_KEY: &str = "adjacency_built";
fn adjacency_key(node: &Uuid, child: &Uuid, parent: &Uuid) -> [u8; 48] {
let mut key = [0u8; 48];
key[..16].copy_from_slice(node.as_bytes());
key[16..32].copy_from_slice(child.as_bytes());
key[32..].copy_from_slice(parent.as_bytes());
key
}
fn adjacency_value(record: &MatchRecord) -> [u8; 20] {
let mut buf = [0u8; 20];
buf[..8].copy_from_slice(&record.jaccard_similarity.to_le_bytes());
buf[8..12].copy_from_slice(&record.size_difference.to_le_bytes());
buf[12..].copy_from_slice(&record.size_difference_pct.to_le_bytes());
buf
}
fn adjacency_entries(record: &MatchRecord) -> Option<([u8; 48], [u8; 48], [u8; 20])> {
if record.child_id == record.parent_id {
return None;
}
let value = adjacency_value(record);
let from_child = adjacency_key(&record.child_id, &record.child_id, &record.parent_id);
let from_parent = adjacency_key(&record.parent_id, &record.child_id, &record.parent_id);
Some((from_child, from_parent, value))
}
const FILTERED_PARENTS_TABLE: TableDefinition<&[u8], &[u8]> =
TableDefinition::new("filtered_parents");
const META_TABLE: TableDefinition<&str, &[u8]> = TableDefinition::new("meta");
const SYNC_STATE_TABLE: TableDefinition<&str, &[u8]> = TableDefinition::new("sync_state");
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SyncStep {
NotStarted = 0,
WritingDupes = 1,
MarkingParents = 2,
MarkingChildren = 3,
Completed = 4,
}
impl From<u8> for SyncStep {
fn from(v: u8) -> Self {
match v {
1 => SyncStep::WritingDupes,
2 => SyncStep::MarkingParents,
3 => SyncStep::MarkingChildren,
4 => SyncStep::Completed,
_ => SyncStep::NotStarted,
}
}
}
#[derive(Debug, Clone)]
pub struct SyncProgress {
pub step: SyncStep,
pub dupes_written: u64,
pub dupes_total: u64,
pub parents_marked: u64,
pub parents_total: u64,
pub children_marked: u64,
pub children_total: u64,
pub started_at: u64,
pub completed_at: u64,
}
#[derive(Debug, Clone, Copy, Default)]
pub struct AdjacencyBuildStats {
pub edges_indexed: u64,
pub entries_written: u64,
}
pub struct MatchStore {
db: Database,
full_scans: AtomicU64,
}
impl MatchStore {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
let db = Database::create(path.as_ref())
.with_context(|| format!("Failed to open matches DB at {:?}", path.as_ref()))?;
let write_txn = db.begin_write()?;
{
let _ = write_txn.open_table(MATCHES_TABLE)?;
let _ = write_txn.open_table(ADJACENCY_TABLE)?;
let _ = write_txn.open_table(META_TABLE)?;
}
write_txn.commit()?;
Ok(Self {
db,
full_scans: AtomicU64::new(0),
})
}
pub fn full_scan_count(&self) -> u64 {
self.full_scans.load(Ordering::Relaxed)
}
pub fn insert(&self, record: &MatchRecord) -> Result<bool> {
Ok(self.insert_batch(std::slice::from_ref(record))? > 0)
}
pub fn insert_batch(&self, records: &[MatchRecord]) -> Result<usize> {
if records.is_empty() {
return Ok(0);
}
let mut inserted = 0;
let write_txn = self.db.begin_write()?;
{
let mut table = write_txn.open_table(MATCHES_TABLE)?;
let mut adjacency = write_txn.open_table(ADJACENCY_TABLE)?;
for record in records {
let key = match_key(record.child_id, record.parent_id);
let should_insert = if let Some(existing) = table.get(key.as_slice())? {
let existing_data = existing.value();
if existing_data.len() >= 24 {
let existing_jaccard = f64::from_le_bytes(
existing_data[16..24].try_into().unwrap_or([0u8; 8]),
);
record.jaccard_similarity > existing_jaccard
} else {
true }
} else {
true };
if should_insert {
let value = self.serialize_record(record);
table.insert(key.as_slice(), value.as_slice())?;
if let Some((from_child, from_parent, av)) = adjacency_entries(record) {
adjacency.insert(from_child.as_slice(), av.as_slice())?;
adjacency.insert(from_parent.as_slice(), av.as_slice())?;
}
inserted += 1;
}
}
}
write_txn.commit()?;
Ok(inserted)
}
pub fn get(&self, child_id: &Uuid) -> Result<Option<MatchRecord>> {
let mut best: Option<MatchRecord> = None;
for record in self.iter()? {
if record.child_id == *child_id {
let replace = best
.as_ref()
.map(|existing| {
let existing_self = existing.child_id == existing.parent_id;
let record_self = record.child_id == record.parent_id;
(existing_self || record.jaccard_similarity > existing.jaccard_similarity)
&& !record_self
})
.unwrap_or(true);
if replace {
best = Some(record);
}
}
}
Ok(best)
}
pub fn contains(&self, child_id: &Uuid) -> Result<bool> {
Ok(self.get(child_id)?.is_some())
}
pub fn count(&self) -> Result<u64> {
let read_txn = self.db.begin_read()?;
let table = read_txn.open_table(MATCHES_TABLE)?;
Ok(table.len()?)
}
pub fn iter(&self) -> Result<Vec<MatchRecord>> {
self.full_scans.fetch_add(1, Ordering::Relaxed);
let read_txn = self.db.begin_read()?;
let table = read_txn.open_table(MATCHES_TABLE)?;
let mut records = Vec::new();
for entry in table.iter()? {
let (key, value) = entry?;
let child_id = child_id_from_match_key(key.value())?;
records.push(self.deserialize_record(&child_id, value.value())?);
}
Ok(records)
}
pub fn iter_real_matches(&self) -> Result<Vec<MatchRecord>> {
self.full_scans.fetch_add(1, Ordering::Relaxed);
let read_txn = self.db.begin_read()?;
let table = read_txn.open_table(MATCHES_TABLE)?;
let mut records = Vec::new();
for entry in table.iter()? {
let (key, value) = entry?;
let child_id = child_id_from_match_key(key.value())?;
let record = self.deserialize_record(&child_id, value.value())?;
if record.child_id != record.parent_id {
records.push(record);
}
}
Ok(records)
}
pub fn get_real_edges_connected_to(&self, seed_doc_ids: &[Uuid]) -> Result<Vec<MatchRecord>> {
if seed_doc_ids.is_empty() {
return Ok(Vec::new());
}
let mut known_docs: std::collections::HashSet<Uuid> =
seed_doc_ids.iter().copied().collect();
let mut included_edges: std::collections::HashSet<(Uuid, Uuid)> =
std::collections::HashSet::new();
let mut records = Vec::new();
loop {
let before = known_docs.len();
self.full_scans.fetch_add(1, Ordering::Relaxed);
let read_txn = self.db.begin_read()?;
let table = read_txn.open_table(MATCHES_TABLE)?;
for entry in table.iter()? {
let (key, value) = entry?;
let child_id = child_id_from_match_key(key.value())?;
let record = self.deserialize_record(&child_id, value.value())?;
if record.child_id == record.parent_id {
continue;
}
if known_docs.contains(&record.child_id) || known_docs.contains(&record.parent_id) {
known_docs.insert(record.child_id);
known_docs.insert(record.parent_id);
if included_edges.insert((record.child_id, record.parent_id)) {
records.push(record);
}
}
}
if known_docs.len() == before {
break;
}
}
Ok(records)
}
pub fn is_adjacency_built(&self) -> Result<bool> {
let read_txn = self.db.begin_read()?;
let table = read_txn.open_table(META_TABLE)?;
Ok(table
.get(ADJACENCY_BUILT_KEY)?
.map(|v| v.value().first().copied() == Some(1))
.unwrap_or(false))
}
fn set_adjacency_built(&self) -> Result<()> {
let write_txn = self.db.begin_write()?;
{
let mut table = write_txn.open_table(META_TABLE)?;
let flag = [1u8];
table.insert(ADJACENCY_BUILT_KEY, flag.as_slice())?;
}
write_txn.commit()?;
Ok(())
}
pub fn adjacency_entry_count(&self) -> Result<u64> {
let read_txn = self.db.begin_read()?;
let table = read_txn.open_table(ADJACENCY_TABLE)?;
Ok(table.len()?)
}
pub fn build_adjacency_index(&self) -> Result<AdjacencyBuildStats> {
const FLUSH_EVERY: usize = 100_000;
let mut buffer: Vec<([u8; 48], [u8; 20])> = Vec::with_capacity(FLUSH_EVERY + 2);
let mut edges_indexed: u64 = 0;
let mut entries_written: u64 = 0;
let read_txn = self.db.begin_read()?;
{
let table = read_txn.open_table(MATCHES_TABLE)?;
for entry in table.iter()? {
let (key, value) = entry?;
let child_id = child_id_from_match_key(key.value())?;
let record = self.deserialize_record(&child_id, value.value())?;
if let Some((from_child, from_parent, av)) = adjacency_entries(&record) {
buffer.push((from_child, av));
buffer.push((from_parent, av));
edges_indexed += 1;
if buffer.len() >= FLUSH_EVERY {
entries_written += self.flush_adjacency(&buffer)? as u64;
buffer.clear();
}
}
}
}
if !buffer.is_empty() {
entries_written += self.flush_adjacency(&buffer)? as u64;
}
self.set_adjacency_built()?;
Ok(AdjacencyBuildStats {
edges_indexed,
entries_written,
})
}
fn flush_adjacency(&self, entries: &[([u8; 48], [u8; 20])]) -> Result<usize> {
let mut inserted = 0;
let write_txn = self.db.begin_write()?;
{
let mut adjacency = write_txn.open_table(ADJACENCY_TABLE)?;
for (k, v) in entries {
if adjacency.get(k.as_slice())?.is_none() {
adjacency.insert(k.as_slice(), v.as_slice())?;
inserted += 1;
}
}
}
write_txn.commit()?;
Ok(inserted)
}
pub fn get_real_edges_connected_to_indexed(
&self,
seed_doc_ids: &[Uuid],
) -> Result<Vec<MatchRecord>> {
if seed_doc_ids.is_empty() {
return Ok(Vec::new());
}
let read_txn = self.db.begin_read()?;
let adjacency = read_txn.open_table(ADJACENCY_TABLE)?;
let mut visited_nodes: std::collections::HashSet<Uuid> =
seed_doc_ids.iter().copied().collect();
let mut stack: Vec<Uuid> = seed_doc_ids.to_vec();
let mut visited_edges: std::collections::HashSet<(Uuid, Uuid)> =
std::collections::HashSet::new();
let mut records = Vec::new();
while let Some(node) = stack.pop() {
let node_bytes = node.as_bytes();
let mut lo = [0u8; 48];
lo[..16].copy_from_slice(node_bytes);
let mut hi = [0xffu8; 48];
hi[..16].copy_from_slice(node_bytes);
for entry in adjacency.range(lo.as_slice()..=hi.as_slice())? {
let (key, value) = entry?;
let key = key.value();
if key.len() != 48 {
continue;
}
let child_id = Uuid::from_slice(&key[16..32])?;
let parent_id = Uuid::from_slice(&key[32..48])?;
if visited_edges.insert((child_id, parent_id)) {
let v = value.value();
if v.len() >= 20 {
records.push(MatchRecord {
child_id,
parent_id,
jaccard_similarity: f64::from_le_bytes(v[0..8].try_into()?),
size_difference: i32::from_le_bytes(v[8..12].try_into()?),
size_difference_pct: f64::from_le_bytes(v[12..20].try_into()?),
});
}
}
let other = if child_id == node {
parent_id
} else {
child_id
};
if visited_nodes.insert(other) {
stack.push(other);
}
}
}
Ok(records)
}
pub fn get_real_edges_connected_to_auto(
&self,
seed_doc_ids: &[Uuid],
) -> Result<(Vec<MatchRecord>, bool)> {
if self.is_adjacency_built()? {
Ok((
self.get_real_edges_connected_to_indexed(seed_doc_ids)?,
true,
))
} else {
Ok((self.get_real_edges_connected_to(seed_doc_ids)?, false))
}
}
pub fn get_unsynced(&self) -> Result<Vec<MatchRecord>> {
self.iter()
}
pub fn get_all(&self) -> Result<Vec<MatchRecord>> {
self.iter()
}
pub fn get_all_real(&self) -> Result<Vec<MatchRecord>> {
self.iter_real_matches()
}
fn serialize_record(&self, record: &MatchRecord) -> Vec<u8> {
let mut buf = Vec::with_capacity(36);
buf.extend_from_slice(record.parent_id.as_bytes());
buf.extend_from_slice(&record.jaccard_similarity.to_le_bytes());
buf.extend_from_slice(&record.size_difference.to_le_bytes());
buf.extend_from_slice(&record.size_difference_pct.to_le_bytes());
buf
}
fn deserialize_record(&self, child_id: &Uuid, data: &[u8]) -> Result<MatchRecord> {
if data.len() < 36 {
anyhow::bail!("Invalid match record data length: {}", data.len());
}
let parent_id = Uuid::from_slice(&data[0..16])?;
let jaccard = f64::from_le_bytes(data[16..24].try_into()?);
let size_diff = i32::from_le_bytes(data[24..28].try_into()?);
let size_diff_pct = f64::from_le_bytes(data[28..36].try_into()?);
Ok(MatchRecord {
child_id: *child_id,
parent_id,
jaccard_similarity: jaccard,
size_difference: size_diff,
size_difference_pct: size_diff_pct,
})
}
}
pub struct FilteredParentStore {
db: Database,
}
impl FilteredParentStore {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
let db = Database::create(path.as_ref())
.with_context(|| format!("Failed to open filtered parent DB at {:?}", path.as_ref()))?;
let write_txn = db.begin_write()?;
{
let _ = write_txn.open_table(FILTERED_PARENTS_TABLE)?;
}
write_txn.commit()?;
Ok(Self { db })
}
pub fn insert_batch(&self, doc_ids: &[Uuid]) -> Result<usize> {
if doc_ids.is_empty() {
return Ok(0);
}
let write_txn = self.db.begin_write()?;
let mut inserted = 0usize;
{
let mut table = write_txn.open_table(FILTERED_PARENTS_TABLE)?;
for doc_id in doc_ids {
let empty: &[u8] = &[];
if table.insert(doc_id.as_bytes().as_slice(), empty)?.is_none() {
inserted += 1;
}
}
}
write_txn.commit()?;
Ok(inserted)
}
pub fn iter(&self) -> Result<Vec<Uuid>> {
let read_txn = self.db.begin_read()?;
let table = read_txn.open_table(FILTERED_PARENTS_TABLE)?;
let mut ids = Vec::with_capacity(table.len()? as usize);
for entry in table.iter()? {
let (key, _) = entry?;
ids.push(Uuid::from_slice(key.value())?);
}
Ok(ids)
}
pub fn count(&self) -> Result<u64> {
let read_txn = self.db.begin_read()?;
let table = read_txn.open_table(FILTERED_PARENTS_TABLE)?;
Ok(table.len()?)
}
}
pub struct StateStore {
db: Database,
}
impl StateStore {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
let db = Database::create(path.as_ref())
.with_context(|| format!("Failed to open state DB at {:?}", path.as_ref()))?;
let write_txn = db.begin_write()?;
{
let _ = write_txn.open_table(META_TABLE)?;
let _ = write_txn.open_table(SYNC_STATE_TABLE)?;
}
write_txn.commit()?;
Ok(Self { db })
}
pub fn get_sync_progress(&self) -> Result<SyncProgress> {
let read_txn = self.db.begin_read()?;
let table = read_txn.open_table(SYNC_STATE_TABLE)?;
let get_u64 = |key: &str| -> u64 {
table
.get(key)
.ok()
.flatten()
.and_then(|v| {
let bytes: [u8; 8] = v.value().try_into().ok()?;
Some(u64::from_le_bytes(bytes))
})
.unwrap_or(0)
};
let step = SyncStep::from(get_u64("step") as u8);
Ok(SyncProgress {
step,
dupes_written: get_u64("dupes_written"),
dupes_total: get_u64("dupes_total"),
parents_marked: get_u64("parents_marked"),
parents_total: get_u64("parents_total"),
children_marked: get_u64("children_marked"),
children_total: get_u64("children_total"),
started_at: get_u64("started_at"),
completed_at: get_u64("completed_at"),
})
}
pub fn set_sync_progress(&self, progress: &SyncProgress) -> Result<()> {
let write_txn = self.db.begin_write()?;
{
let mut table = write_txn.open_table(SYNC_STATE_TABLE)?;
let set_u64 = |t: &mut redb::Table<&str, &[u8]>, key: &str, val: u64| -> Result<()> {
t.insert(key, val.to_le_bytes().as_slice())?;
Ok(())
};
set_u64(&mut table, "step", progress.step as u64)?;
set_u64(&mut table, "dupes_written", progress.dupes_written)?;
set_u64(&mut table, "dupes_total", progress.dupes_total)?;
set_u64(&mut table, "parents_marked", progress.parents_marked)?;
set_u64(&mut table, "parents_total", progress.parents_total)?;
set_u64(&mut table, "children_marked", progress.children_marked)?;
set_u64(&mut table, "children_total", progress.children_total)?;
set_u64(&mut table, "started_at", progress.started_at)?;
set_u64(&mut table, "completed_at", progress.completed_at)?;
}
write_txn.commit()?;
Ok(())
}
pub fn update_sync_field(&self, field: &str, value: u64) -> Result<()> {
let write_txn = self.db.begin_write()?;
{
let mut table = write_txn.open_table(SYNC_STATE_TABLE)?;
table.insert(field, value.to_le_bytes().as_slice())?;
}
write_txn.commit()?;
Ok(())
}
pub fn reset_sync_progress(&self) -> Result<()> {
let write_txn = self.db.begin_write()?;
{
let mut table = write_txn.open_table(SYNC_STATE_TABLE)?;
let keys: Vec<String> = table
.iter()?
.map(|r| r.map(|(k, _)| k.value().to_string()))
.collect::<Result<_, _>>()?;
for key in keys {
table.remove(key.as_str())?;
}
}
write_txn.commit()?;
Ok(())
}
pub fn is_sync_in_progress(&self) -> Result<bool> {
let progress = self.get_sync_progress()?;
Ok(progress.step != SyncStep::NotStarted && progress.step != SyncStep::Completed)
}
pub fn is_sync_completed(&self) -> Result<bool> {
let progress = self.get_sync_progress()?;
Ok(progress.step == SyncStep::Completed)
}
}
pub struct DatasetStorage {
pub matches: MatchStore,
pub base_path: std::path::PathBuf,
}
impl DatasetStorage {
pub fn open<P: AsRef<Path>>(base_path: P) -> Result<Self> {
let base = base_path.as_ref();
std::fs::create_dir_all(base)?;
let matches_path = base.join("matches.redb");
Ok(Self {
matches: MatchStore::open(&matches_path)?,
base_path: base.to_path_buf(),
})
}
pub fn open_for_dataset(data_dir: &Path, dataset_id: &Uuid) -> Result<Self> {
let path = data_dir.join(format!("dataset_{}", dataset_id));
Self::open(path)
}
pub fn match_count(&self) -> Result<u64> {
self.matches.count()
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
#[test]
fn test_match_store() {
let dir = tempdir().unwrap();
let store = MatchStore::open(dir.path().join("matches.redb")).unwrap();
let child_id = Uuid::new_v4();
let parent_id = Uuid::new_v4();
let record = MatchRecord {
child_id,
parent_id,
jaccard_similarity: 0.85,
size_difference: 100,
size_difference_pct: 0.1,
};
assert!(store.insert(&record).unwrap());
let retrieved = store.get(&child_id).unwrap().unwrap();
assert_eq!(retrieved.parent_id, parent_id);
assert!((retrieved.jaccard_similarity - 0.85).abs() < 0.001);
assert_eq!(retrieved.size_difference, 100);
}
#[test]
fn test_match_store_preserves_raw_edges_and_get_returns_best() {
let dir = tempdir().unwrap();
let store = MatchStore::open(dir.path().join("matches.redb")).unwrap();
let child_id = Uuid::new_v4();
let parent_a = Uuid::new_v4();
let parent_b = Uuid::new_v4();
let parent_c = Uuid::new_v4();
let record_a = MatchRecord {
child_id,
parent_id: parent_a,
jaccard_similarity: 0.80,
size_difference: 100,
size_difference_pct: 0.1,
};
assert!(store.insert(&record_a).unwrap());
let record_b = MatchRecord {
child_id,
parent_id: parent_b,
jaccard_similarity: 0.70,
size_difference: 50,
size_difference_pct: 0.05,
};
assert!(store.insert(&record_b).unwrap());
assert_eq!(store.count().unwrap(), 2);
let retrieved = store.get(&child_id).unwrap().unwrap();
assert_eq!(retrieved.parent_id, parent_a);
assert!((retrieved.jaccard_similarity - 0.80).abs() < 0.001);
let record_c = MatchRecord {
child_id,
parent_id: parent_c,
jaccard_similarity: 0.95,
size_difference: 20,
size_difference_pct: 0.02,
};
assert!(store.insert(&record_c).unwrap());
assert_eq!(store.count().unwrap(), 3);
let retrieved = store.get(&child_id).unwrap().unwrap();
assert_eq!(retrieved.parent_id, parent_c);
assert!((retrieved.jaccard_similarity - 0.95).abs() < 0.001);
}
#[test]
fn test_match_store_real_match_replaces_legacy_self_parent() {
let dir = tempdir().unwrap();
let store = MatchStore::open(dir.path().join("matches.redb")).unwrap();
let child_id = Uuid::new_v4();
let real_parent = Uuid::new_v4();
let legacy_self_parent = MatchRecord {
child_id,
parent_id: child_id,
jaccard_similarity: 1.0,
size_difference: 0,
size_difference_pct: 0.0,
};
assert!(store.insert(&legacy_self_parent).unwrap());
let real_match = MatchRecord {
child_id,
parent_id: real_parent,
jaccard_similarity: 0.85,
size_difference: 100,
size_difference_pct: 0.1,
};
assert!(store.insert(&real_match).unwrap());
let retrieved = store.get(&child_id).unwrap().unwrap();
assert_eq!(retrieved.parent_id, real_parent);
assert!((retrieved.jaccard_similarity - 0.85).abs() < 0.001);
}
#[test]
fn test_match_store_loads_only_connected_edges() {
let dir = tempdir().unwrap();
let store = MatchStore::open(dir.path().join("matches.redb")).unwrap();
let a = Uuid::from_u128(1);
let b = Uuid::from_u128(2);
let c = Uuid::from_u128(3);
let x = Uuid::from_u128(10);
let y = Uuid::from_u128(11);
let mk = |child_id, parent_id| MatchRecord {
child_id,
parent_id,
jaccard_similarity: 0.9,
size_difference: 1,
size_difference_pct: 0.01,
};
store.insert_batch(&[mk(a, b), mk(b, c), mk(x, y)]).unwrap();
let connected = store.get_real_edges_connected_to(&[a]).unwrap();
let edges: std::collections::HashSet<_> = connected
.iter()
.map(|m| (m.child_id, m.parent_id))
.collect();
assert_eq!(edges.len(), 2);
assert!(edges.contains(&(a, b)));
assert!(edges.contains(&(b, c)));
assert!(!edges.contains(&(x, y)));
}
#[test]
fn test_filtered_parent_store_round_trip() {
let dir = tempdir().unwrap();
let store = FilteredParentStore::open(dir.path().join("filtered_parents.redb")).unwrap();
let a = Uuid::from_u128(1);
let b = Uuid::from_u128(2);
assert_eq!(store.insert_batch(&[a, b, a]).unwrap(), 2);
assert_eq!(store.count().unwrap(), 2);
let ids: std::collections::HashSet<_> = store.iter().unwrap().into_iter().collect();
assert_eq!(ids, [a, b].into_iter().collect());
}
#[test]
fn test_dataset_storage() {
let dir = tempdir().unwrap();
let storage = DatasetStorage::open(dir.path()).unwrap();
let child_id = Uuid::new_v4();
let parent_id = Uuid::new_v4();
let record = MatchRecord {
child_id,
parent_id,
jaccard_similarity: 0.9,
size_difference: 50,
size_difference_pct: 0.05,
};
assert!(storage.matches.insert(&record).unwrap());
assert_eq!(storage.match_count().unwrap(), 1);
assert!(storage.matches.contains(&child_id).unwrap());
}
#[test]
fn test_sync_progress_reset_must_be_readable_as_not_started() {
let dir = tempdir().unwrap();
let store = StateStore::open(dir.path().join("state.redb")).unwrap();
let progress = SyncProgress {
step: SyncStep::Completed,
dupes_written: 1000,
dupes_total: 1000,
parents_marked: 500,
parents_total: 500,
children_marked: 500,
children_total: 500,
started_at: 12345,
completed_at: 12400,
};
store.set_sync_progress(&progress).unwrap();
let read1 = store.get_sync_progress().unwrap();
assert_eq!(read1.step, SyncStep::Completed);
store.reset_sync_progress().unwrap();
let read2 = store.get_sync_progress().unwrap();
assert_eq!(
read2.step,
SyncStep::NotStarted,
"After reset, sync progress must read as NotStarted"
);
assert_eq!(read2.dupes_written, 0);
assert_eq!(read2.parents_marked, 0);
assert_eq!(read2.children_marked, 0);
}
#[test]
fn test_sync_progress_persistence() {
let dir = tempdir().unwrap();
let path = dir.path().join("state.redb");
{
let store = StateStore::open(&path).unwrap();
let progress = SyncProgress {
step: SyncStep::MarkingChildren,
dupes_written: 1000,
dupes_total: 1000,
parents_marked: 500,
parents_total: 500,
children_marked: 250,
children_total: 500,
started_at: 12345,
completed_at: 0,
};
store.set_sync_progress(&progress).unwrap();
}
{
let store = StateStore::open(&path).unwrap();
let read = store.get_sync_progress().unwrap();
assert_eq!(read.step, SyncStep::MarkingChildren);
assert_eq!(read.children_marked, 250);
assert_eq!(read.children_total, 500);
}
}
}