use super::super::change_count;
use super::{
canonical, capture, identity::Resolver, open, publication::build, row, setup,
ManagedConnection, StorageReadControl, FIELD, TABLE,
};
use uqa_storage::diskann_index::{
changes::{DiskANNPruneCursor, DiskANNPruneRequest, DiskANNPruneResult},
format::DiskANNGeneration,
};
use uqa_storage::key_value::KeyValueDiskANNPruner;
fn page(max_records: usize) -> DiskANNPruneRequest {
DiskANNPruneRequest {
after: None,
max_records,
}
}
fn prune(
connection: &ManagedConnection,
pruner: &KeyValueDiskANNPruner,
request: DiskANNPruneRequest,
control: &StorageReadControl,
) -> uqa_storage::StorageBackendResult<DiskANNPruneResult> {
connection.prune_diskann_changes(
&capture(connection, control),
&Resolver,
pruner,
request,
control,
)
}
#[test]
fn native_diskann_pruning_preserves_late_writes_retained_views_and_cold_reopen() {
for mode in 0..4 {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("pruning.db");
let generation = {
let connection = open(&path, mode);
let peer = open(&path, mode);
let catalog = setup(&connection);
let mut row = row();
row.definition_json = Some(serde_json::to_string(&[82; 16]).unwrap());
catalog.save_catalog_index_row(&row).unwrap();
let control = StorageReadControl::with_limit(1 << 20);
let canonical = canonical(&connection, TABLE, FIELD, 2);
let original = canonical.replace(1, &[vec![1.0, 0.0]], &control).unwrap();
canonical.replace(2, &[], &control).unwrap();
canonical.replace(3, &[], &control).unwrap();
canonical.replace(3, &[vec![0.0, 1.0]], &control).unwrap();
peer.begin_transaction().unwrap();
let late = super::canonical(&peer, TABLE, FIELD, 2)
.replace(4, &[], &control)
.unwrap();
let (coverage, stage) = build(&connection, &control);
let repository = connection.diskann_generations(&control).unwrap();
let pruner = KeyValueDiskANNPruner::open(
repository
.open_source(stage.generation(), &control)
.unwrap(),
8192,
&control,
)
.unwrap();
connection.begin_transaction().unwrap();
connection
.publish_diskann_generation(&coverage, &Resolver, &control)
.unwrap();
assert!(prune(&connection, &pruner, page(64), &control).is_err());
connection.rollback_transaction().unwrap();
connection
.publish_diskann_generation(&coverage, &Resolver, &control)
.unwrap();
let changed = canonical.replace(1, &[vec![0.0, 1.0]], &control).unwrap();
peer.commit_transaction().unwrap();
let old_cursor = prune_pass(&connection, &canonical, &pruner, &control);
assert_eq!(
coverage.source().origin(1, &control).unwrap(),
Some(original)
);
assert_eq!(
coverage
.source()
.next_change_after(None, &control)
.unwrap()
.unwrap()
.document(),
1
);
let current = capture(&connection, &control);
assert_eq!(
current
.next_change_after(None, &control)
.unwrap()
.unwrap()
.version(),
changed
);
assert_eq!(
current
.next_change_after(Some(1), &control)
.unwrap()
.unwrap()
.version(),
late
);
publication_races(&connection, &peer, &pruner, old_cursor, &control)
};
verify_reopen(&open(&path, mode), generation);
}
}
fn prune_pass(
connection: &ManagedConnection,
canonical: &crate::vector_index::SQLiteDiskANNCanonical,
pruner: &KeyValueDiskANNPruner,
control: &StorageReadControl,
) -> Option<DiskANNPruneCursor> {
assert_eq!(change_count(connection), 6);
connection.begin_transaction().unwrap();
connection.savepoint("pruning").unwrap();
assert_eq!(
prune(connection, pruner, page(64), control)
.unwrap()
.removed,
4
);
connection.rollback_to_savepoint("pruning").unwrap();
canonical.replace(1, &[], control).unwrap();
assert!(prune(connection, pruner, page(64), control).is_err());
connection.rollback_transaction().unwrap();
assert_eq!(change_count(connection), 6);
let mut request = page(1);
let mut totals = (0, 0);
let old_cursor;
loop {
let result = prune(connection, pruner, request, control).unwrap();
assert!(result.examined <= 1 && result.removed <= 1);
totals.0 += result.examined;
totals.1 += result.removed;
if result.next.is_none() {
old_cursor = request.after;
break;
}
request.after = result.next;
}
assert_eq!(totals, (6, 4));
old_cursor
}
fn publication_races(
connection: &ManagedConnection,
peer: &ManagedConnection,
pruner: &KeyValueDiskANNPruner,
old_cursor: Option<DiskANNPruneCursor>,
control: &StorageReadControl,
) -> DiskANNGeneration {
let canonical = canonical(connection, TABLE, FIELD, 2);
let repository = connection.diskann_generations(control).unwrap();
let mut definition = row();
definition.definition_json = Some(serde_json::to_string(&[82; 16]).unwrap());
canonical.replace(0, &[], control).unwrap();
canonical.replace(0, &[], control).unwrap();
assert_eq!(
prune(
connection,
pruner,
DiskANNPruneRequest {
after: old_cursor,
max_records: 64
},
control
)
.unwrap()
.examined,
0
);
let (new_coverage, new_stage) = build(connection, control);
connection.begin_transaction().unwrap();
assert_eq!(
prune(connection, pruner, page(64), control)
.unwrap()
.removed,
1
);
peer.publish_diskann_generation(&new_coverage, &Resolver, control)
.unwrap();
assert!(connection.commit_transaction().is_err());
connection.rollback_transaction().unwrap();
assert_eq!(change_count(connection), 4);
assert!(prune(connection, pruner, page(64), control).is_err());
let pruner = KeyValueDiskANNPruner::open(
repository
.open_source(new_stage.generation(), control)
.unwrap(),
8192,
control,
)
.unwrap();
assert!(prune(
connection,
&pruner,
DiskANNPruneRequest {
after: old_cursor,
max_records: 64
},
control
)
.is_err());
connection.begin_transaction().unwrap();
prune(connection, &pruner, page(64), control).unwrap();
crate::Catalog::open(peer.clone())
.unwrap()
.save_catalog_index_row(&definition)
.unwrap();
assert!(connection.commit_transaction().is_err());
connection.rollback_transaction().unwrap();
assert_eq!(change_count(connection), 4);
assert!(prune(connection, &pruner, page(0), control).is_err());
let tiny = StorageReadControl::with_limit(1);
assert!(connection
.prune_diskann_changes(
&capture(connection, control),
&Resolver,
&pruner,
page(64),
&tiny
)
.is_err());
let cancelled = StorageReadControl::with_limit(1 << 20);
cancelled.cancellation().cancel();
assert!(connection
.prune_diskann_changes(
&capture(connection, control),
&Resolver,
&pruner,
page(64),
&cancelled
)
.is_err());
canonical.replace(9, &[], control).unwrap();
assert_eq!(
prune(connection, &pruner, page(64), control)
.unwrap()
.removed,
4
);
assert_eq!(change_count(connection), 1);
new_stage.generation()
}
fn verify_reopen(connection: &ManagedConnection, generation: DiskANNGeneration) {
let control = StorageReadControl::with_limit(1 << 20);
let repository = connection.diskann_generations(&control).unwrap();
let pruner = KeyValueDiskANNPruner::open(
repository.open_source(generation, &control).unwrap(),
8192,
&control,
)
.unwrap();
let result = prune(connection, &pruner, page(64), &control).unwrap();
assert_eq!((result.examined, result.removed, result.next), (1, 0, None));
assert_eq!(
capture(connection, &control)
.next_change_after(None, &control)
.unwrap()
.unwrap()
.document(),
9
);
assert_eq!(change_count(connection), 1);
finite_discovery(connection, &control);
clear_racing_pruning(connection, &control);
}
fn clear_racing_pruning(connection: &ManagedConnection, control: &StorageReadControl) {
use uqa_storage::{diskann_index::build::DiskANNTemporaryBudget, VectorIndex};
let temporary = DiskANNTemporaryBudget::new(1 << 20);
for refresh in [false, true] {
let canonical = canonical(connection, TABLE, FIELD, 2);
canonical.replace(0, &[], control).unwrap();
canonical.replace(0, &[], control).unwrap();
let peer = connection.new_controlled_session(control).unwrap();
let pruner = super::canonical(&peer, TABLE, FIELD, 2)
.journal_pruner(
&row().relation,
std::sync::Arc::new(Resolver),
8192,
control,
)
.unwrap();
let mut index = super::runtime::runtime(connection, &temporary, control);
let retained = index.snapshot().unwrap();
let previous_count = retained.count().unwrap();
connection.begin_transaction().unwrap();
index.clear().unwrap();
peer.begin_transaction().unwrap();
assert!(pruner.prune(page(64), control).unwrap().removed > 0);
peer.commit_transaction().unwrap();
if refresh {
connection
.refresh_transaction_snapshot(control.cancellation())
.unwrap();
}
connection.commit_transaction().unwrap();
assert_eq!(index.count().unwrap(), 0);
assert_eq!(retained.count().unwrap(), previous_count);
assert_eq!(change_count(connection), 0);
}
}
fn finite_discovery(connection: &ManagedConnection, control: &StorageReadControl) {
let original = StorageReadControl::new(control.memory(), &uqa_core::CancellationToken::new());
let pruner = canonical(connection, TABLE, FIELD, 2)
.journal_pruner(
&row().relation,
std::sync::Arc::new(Resolver),
8192,
&original,
)
.unwrap();
assert!(pruner.prune(page(64), control).is_err());
let canonical = canonical(connection, TABLE, FIELD, 2);
let retained = canonical.retain(control).unwrap();
canonical.replace(9, &[], control).unwrap();
connection.begin_transaction().unwrap();
let result = pruner.prune(page(64), control).unwrap();
assert_eq!((result.examined, result.removed, result.next), (1, 1, None));
connection.commit_transaction().unwrap();
assert_eq!(change_count(connection), 1);
assert!(retained.next_change_after(None, control).unwrap().is_some());
original.cancellation().cancel();
connection.begin_transaction().unwrap();
assert!(pruner.prune(page(64), control).is_err());
connection.rollback_transaction().unwrap();
}