use std::collections::{HashMap, HashSet};
use std::path::Path;
use std::sync::Arc;
use redb::{Database, ReadableDatabase, ReadableTable, TableDefinition};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use super::{
Cache, CacheError, CachedTarget, PendingItem, PendingState, Queue, QueueError, SearchHit,
UpsertOutcome, RANK_EXACT, RANK_PREFIX, RANK_SUBSTRING,
};
use crate::identity::target_id_from_designation;
use crate::normalize::normalize;
use crate::types::{AliasKind, ObjectType, ResolvedAlias, ResolvedIdentity, TargetSource};
const TARGETS: TableDefinition<&str, &[u8]> = TableDefinition::new("targets");
const ALIASES: TableDefinition<&str, &[u8]> = TableDefinition::new("aliases");
const PENDING: TableDefinition<&str, &[u8]> = TableDefinition::new("pending");
const META: TableDefinition<&str, u64> = TableDefinition::new("meta");
const PENDING_SEQ_KEY: &str = "pending_seq";
fn cache_err<E: std::fmt::Display>(e: E) -> CacheError {
CacheError::Backend(e.to_string())
}
fn queue_err<E: std::fmt::Display>(e: E) -> QueueError {
QueueError::Backend(e.to_string())
}
fn now_iso() -> String {
time::OffsetDateTime::now_utc()
.format(&time::format_description::well_known::Rfc3339)
.expect("Rfc3339 format is always valid for OffsetDateTime")
}
#[derive(Serialize, Deserialize)]
struct StoredTarget {
simbad_oid: Option<i64>,
primary_designation: String,
common_name: Option<String>,
object_type: ObjectType,
otype_raw: String,
ra_deg: f64,
dec_deg: f64,
#[serde(default)]
v_mag: Option<f64>,
source: TargetSource,
resolved_at: String,
}
impl StoredTarget {
fn from_identity(identity: &ResolvedIdentity) -> Self {
Self {
simbad_oid: identity.simbad_oid,
primary_designation: identity.primary_designation.clone(),
common_name: identity.common_name.clone(),
object_type: identity.object_type,
otype_raw: identity.otype_raw.clone(),
ra_deg: identity.ra_deg,
dec_deg: identity.dec_deg,
v_mag: identity.v_mag,
source: identity.source,
resolved_at: now_iso(),
}
}
fn into_target(self, id: Uuid, aliases: Vec<ResolvedAlias>) -> CachedTarget {
CachedTarget {
id,
simbad_oid: self.simbad_oid,
primary_designation: self.primary_designation,
common_name: self.common_name,
object_type: self.object_type,
otype_raw: self.otype_raw,
ra_deg: self.ra_deg,
dec_deg: self.dec_deg,
v_mag: self.v_mag,
source: self.source,
resolved_at: self.resolved_at,
aliases,
}
}
}
#[derive(Serialize, Deserialize)]
struct StoredAlias {
target_id: String,
alias: ResolvedAlias,
}
#[derive(Serialize, Deserialize)]
struct StoredPending {
query: String,
state: String,
attempts: i64,
target_id: Option<String>,
seq: u64,
}
impl StoredPending {
fn into_item(self, id: String) -> Result<PendingItem, QueueError> {
let state = PendingState::from_wire(&self.state)
.ok_or_else(|| QueueError::InvalidState(self.state.clone()))?;
let target_id = self
.target_id
.map(|t| Uuid::parse_str(&t).map_err(|e| QueueError::InvalidUuid(t.clone(), e)))
.transpose()?;
Ok(PendingItem { id, query: self.query, state, attempts: self.attempts, target_id })
}
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub enum BatchDurability {
#[default]
Durable,
Eventual,
}
#[derive(Clone)]
pub struct Store {
db: Arc<Database>,
bulk_durability: BatchDurability,
}
impl Store {
pub fn open(path: impl AsRef<Path>) -> Result<Self, CacheError> {
Self::open_with(path, BatchDurability::default())
}
pub fn open_with(
path: impl AsRef<Path>,
bulk_durability: BatchDurability,
) -> Result<Self, CacheError> {
let db = Database::create(path).map_err(cache_err)?;
init_tables(&db)?;
Ok(Self { db: Arc::new(db), bulk_durability })
}
pub fn in_memory() -> Result<Self, CacheError> {
let db = Database::builder()
.create_with_backend(redb::backends::InMemoryBackend::new())
.map_err(cache_err)?;
init_tables(&db)?;
Ok(Self { db: Arc::new(db), bulk_durability: BatchDurability::default() })
}
#[must_use]
pub fn cache(&self) -> RedbCache {
RedbCache { db: self.db.clone(), bulk_durability: self.bulk_durability }
}
#[must_use]
pub fn queue(&self) -> RedbQueue {
RedbQueue { db: self.db.clone() }
}
}
fn init_tables(db: &Database) -> Result<(), CacheError> {
let w = db.begin_write().map_err(cache_err)?;
{
w.open_table(TARGETS).map_err(cache_err)?;
w.open_table(ALIASES).map_err(cache_err)?;
w.open_table(PENDING).map_err(cache_err)?;
w.open_table(META).map_err(cache_err)?;
}
w.commit().map_err(cache_err)?;
Ok(())
}
#[derive(Clone)]
pub struct RedbCache {
db: Arc<Database>,
bulk_durability: BatchDurability,
}
#[derive(Clone)]
pub struct RedbQueue {
db: Arc<Database>,
}
fn aliases_for(db: &Database, target_id: &str) -> Result<Vec<ResolvedAlias>, CacheError> {
let txn = db.begin_read().map_err(cache_err)?;
let table = txn.open_table(ALIASES).map_err(cache_err)?;
let mut aliases = Vec::new();
for entry in table.iter().map_err(cache_err)? {
let (_, v) = entry.map_err(cache_err)?;
let sa: StoredAlias = serde_json::from_slice(v.value()).map_err(cache_err)?;
if sa.target_id == target_id {
aliases.push(sa.alias);
}
}
aliases.sort_by(|a, b| a.alias.cmp(&b.alias));
Ok(aliases)
}
fn get_by_id(db: &Database, id: Uuid) -> Result<Option<CachedTarget>, CacheError> {
let id_str = id.to_string();
let txn = db.begin_read().map_err(cache_err)?;
let targets = txn.open_table(TARGETS).map_err(cache_err)?;
let Some(bytes) = targets.get(id_str.as_str()).map_err(cache_err)? else {
return Ok(None);
};
let st: StoredTarget = serde_json::from_slice(bytes.value()).map_err(cache_err)?;
drop(bytes);
let aliases = aliases_for(db, &id_str)?;
Ok(Some(st.into_target(id, aliases)))
}
fn get_by_simbad_oid(db: &Database, oid: i64) -> Result<Option<CachedTarget>, CacheError> {
let found = {
let txn = db.begin_read().map_err(cache_err)?;
let targets = txn.open_table(TARGETS).map_err(cache_err)?;
let mut hit: Option<String> = None;
for entry in targets.iter().map_err(cache_err)? {
let (k, v) = entry.map_err(cache_err)?;
let st: StoredTarget = serde_json::from_slice(v.value()).map_err(cache_err)?;
if st.simbad_oid == Some(oid) {
hit = Some(k.value().to_string());
break;
}
}
hit
};
match found {
None => Ok(None),
Some(id_str) => {
let id = Uuid::parse_str(&id_str).map_err(|e| CacheError::InvalidUuid(id_str, e))?;
get_by_id(db, id)
}
}
}
fn get_by_normalized(db: &Database, normalized: &str) -> Result<Option<CachedTarget>, CacheError> {
let found = {
let txn = db.begin_read().map_err(cache_err)?;
let table = txn.open_table(ALIASES).map_err(cache_err)?;
let mut hit: Option<String> = None;
for entry in table.iter().map_err(cache_err)? {
let (_, v) = entry.map_err(cache_err)?;
let sa: StoredAlias = serde_json::from_slice(v.value()).map_err(cache_err)?;
if sa.alias.normalized == normalized {
hit = Some(sa.target_id);
break;
}
}
hit
};
match found {
None => Ok(None),
Some(id_str) => {
let id = Uuid::parse_str(&id_str).map_err(|e| CacheError::InvalidUuid(id_str, e))?;
get_by_id(db, id)
}
}
}
struct Best {
alias: String,
normalized_len: usize,
rank: u8,
}
impl Best {
fn is_better_than(&self, other: &Self) -> bool {
(self.rank, self.normalized_len) < (other.rank, other.normalized_len)
}
}
fn search(db: &Database, query: &str, limit: usize) -> Result<Vec<SearchHit>, CacheError> {
let q = normalize(query);
if q.is_empty() || limit == 0 {
return Ok(Vec::new());
}
let mut best_by_target: HashMap<String, Best> = HashMap::new();
{
let txn = db.begin_read().map_err(cache_err)?;
let table = txn.open_table(ALIASES).map_err(cache_err)?;
for entry in table.iter().map_err(cache_err)? {
let (_, v) = entry.map_err(cache_err)?;
let sa: StoredAlias = serde_json::from_slice(v.value()).map_err(cache_err)?;
let normalized = &sa.alias.normalized;
let rank = if *normalized == q {
RANK_EXACT
} else if normalized.starts_with(&q) {
RANK_PREFIX
} else if normalized.contains(&q) {
RANK_SUBSTRING
} else {
continue;
};
let candidate =
Best { alias: sa.alias.alias.clone(), normalized_len: normalized.len(), rank };
match best_by_target.entry(sa.target_id) {
std::collections::hash_map::Entry::Occupied(mut e) => {
if candidate.is_better_than(e.get()) {
e.insert(candidate);
}
}
std::collections::hash_map::Entry::Vacant(e) => {
e.insert(candidate);
}
}
}
}
let mut ranked: Vec<(String, Best)> = best_by_target.into_iter().collect();
ranked.sort_by(|(_, a), (_, b)| {
(a.rank, a.normalized_len, a.alias.as_str()).cmp(&(
b.rank,
b.normalized_len,
b.alias.as_str(),
))
});
ranked.truncate(limit);
let mut hits = Vec::with_capacity(ranked.len());
for (target_id, best) in ranked {
let id = Uuid::parse_str(&target_id).map_err(|e| CacheError::InvalidUuid(target_id, e))?;
if let Some(target) = get_by_id(db, id)? {
hits.push(SearchHit { target, matched_alias: best.alias, rank: best.rank });
}
}
Ok(hits)
}
enum AliasLookup<'a> {
Scan,
Index(&'a mut HashMap<String, Vec<String>>),
}
fn rewrite_aliases(
w: &redb::WriteTransaction,
target_id: &str,
identity: &ResolvedIdentity,
alias_lookup: &mut AliasLookup,
) -> Result<(), CacheError> {
let mut table = w.open_table(ALIASES).map_err(cache_err)?;
let stale: Vec<String> = match alias_lookup {
AliasLookup::Scan => {
#[cfg(test)]
FULL_TABLE_SCAN_COUNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let mut keys = Vec::new();
for entry in table.iter().map_err(cache_err)? {
let (k, v) = entry.map_err(cache_err)?;
let sa: StoredAlias = serde_json::from_slice(v.value()).map_err(cache_err)?;
if sa.target_id == target_id {
keys.push(k.value().to_string());
}
}
keys
}
AliasLookup::Index(index) => index.remove(target_id).unwrap_or_default(),
};
for k in &stale {
table.remove(k.as_str()).map_err(cache_err)?;
}
let mut aliases = identity.aliases.clone();
let primary_norm = normalize(&identity.primary_designation);
if !aliases.iter().any(|a| a.normalized == primary_norm) {
aliases
.push(ResolvedAlias::new(identity.primary_designation.clone(), AliasKind::Designation));
}
let mut seen = HashSet::with_capacity(aliases.len());
let mut written = Vec::with_capacity(aliases.len());
for alias in aliases {
if !seen.insert(alias.normalized.clone()) {
continue; }
let alias_id = Uuid::new_v4().to_string();
let sa = StoredAlias { target_id: target_id.to_owned(), alias };
let bytes = serde_json::to_vec(&sa).map_err(cache_err)?;
table.insert(alias_id.as_str(), bytes.as_slice()).map_err(cache_err)?;
written.push(alias_id);
}
if let AliasLookup::Index(index) = alias_lookup {
index.insert(target_id.to_owned(), written);
}
Ok(())
}
fn build_alias_index(
w: &redb::WriteTransaction,
) -> Result<HashMap<String, Vec<String>>, CacheError> {
#[cfg(test)]
FULL_TABLE_SCAN_COUNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let table = w.open_table(ALIASES).map_err(cache_err)?;
let mut index: HashMap<String, Vec<String>> = HashMap::new();
for entry in table.iter().map_err(cache_err)? {
let (k, v) = entry.map_err(cache_err)?;
let sa: StoredAlias = serde_json::from_slice(v.value()).map_err(cache_err)?;
index.entry(sa.target_id).or_default().push(k.value().to_string());
}
Ok(index)
}
#[cfg(test)]
static FULL_TABLE_SCAN_COUNT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
enum OidLookup<'a> {
Scan,
Index(&'a mut HashMap<i64, (String, TargetSource)>),
}
fn upsert_within(
w: &redb::WriteTransaction,
identity: &ResolvedIdentity,
namespace: &Uuid,
oid_lookup: &mut OidLookup,
alias_lookup: &mut AliasLookup,
) -> Result<(Uuid, UpsertOutcome), CacheError> {
let derived_id =
target_id_from_designation(namespace, &identity.primary_designation).to_string();
let (id_str, outcome, wrote_row, prior_oid) = {
let mut targets = w.open_table(TARGETS).map_err(cache_err)?;
let mut existing: Option<(String, TargetSource, Option<i64>)> = None;
if let Some(oid) = identity.simbad_oid {
existing = match oid_lookup {
OidLookup::Scan => {
#[cfg(test)]
FULL_TABLE_SCAN_COUNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let mut found = None;
for entry in targets.iter().map_err(cache_err)? {
let (k, v) = entry.map_err(cache_err)?;
let st: StoredTarget =
serde_json::from_slice(v.value()).map_err(cache_err)?;
if st.simbad_oid == Some(oid) {
found = Some((k.value().to_string(), st.source, Some(oid)));
break;
}
}
found
}
OidLookup::Index(index) => {
index.get(&oid).map(|(id, source)| (id.clone(), *source, Some(oid)))
}
};
}
if existing.is_none() {
if let Some(v) = targets.get(derived_id.as_str()).map_err(cache_err)? {
let st: StoredTarget = serde_json::from_slice(v.value()).map_err(cache_err)?;
existing = Some((derived_id.clone(), st.source, st.simbad_oid));
}
}
match existing {
Some((id, source, _)) if !identity.source.may_overwrite(source) => {
(id, UpsertOutcome::SkippedUserOverride, false, None)
}
Some((id, _, prior_oid)) => {
let bytes = serde_json::to_vec(&StoredTarget::from_identity(identity))
.map_err(cache_err)?;
targets.insert(id.as_str(), bytes.as_slice()).map_err(cache_err)?;
(id, UpsertOutcome::Updated, true, prior_oid)
}
None => {
let bytes = serde_json::to_vec(&StoredTarget::from_identity(identity))
.map_err(cache_err)?;
targets.insert(derived_id.as_str(), bytes.as_slice()).map_err(cache_err)?;
(derived_id.clone(), UpsertOutcome::Inserted, true, None)
}
}
};
if wrote_row {
rewrite_aliases(w, &id_str, identity, alias_lookup)?;
if let OidLookup::Index(index) = oid_lookup {
if prior_oid != identity.simbad_oid {
if let Some(stale) = prior_oid {
index.remove(&stale);
}
}
if let Some(oid) = identity.simbad_oid {
index.insert(oid, (id_str.clone(), identity.source));
}
}
}
let id = Uuid::parse_str(&id_str).map_err(|e| CacheError::InvalidUuid(id_str, e))?;
Ok((id, outcome))
}
fn build_oid_index(
w: &redb::WriteTransaction,
) -> Result<HashMap<i64, (String, TargetSource)>, CacheError> {
#[cfg(test)]
FULL_TABLE_SCAN_COUNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let targets = w.open_table(TARGETS).map_err(cache_err)?;
let mut index = HashMap::new();
for entry in targets.iter().map_err(cache_err)? {
let (k, v) = entry.map_err(cache_err)?;
let st: StoredTarget = serde_json::from_slice(v.value()).map_err(cache_err)?;
if let Some(oid) = st.simbad_oid {
index.insert(oid, (k.value().to_string(), st.source));
}
}
Ok(index)
}
#[cfg(test)]
static UPSERT_WRITE_TXN_COUNT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
fn begin_upsert_txn(db: &Database) -> Result<redb::WriteTransaction, CacheError> {
#[cfg(test)]
UPSERT_WRITE_TXN_COUNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
db.begin_write().map_err(cache_err)
}
fn upsert(
db: &Database,
identity: &ResolvedIdentity,
namespace: &Uuid,
) -> Result<(Uuid, UpsertOutcome), CacheError> {
let w = begin_upsert_txn(db)?;
let result =
upsert_within(&w, identity, namespace, &mut OidLookup::Scan, &mut AliasLookup::Scan)?;
w.commit().map_err(cache_err)?;
Ok(result)
}
fn upsert_batch(
db: &Database,
identities: &[ResolvedIdentity],
namespace: &Uuid,
durability: BatchDurability,
) -> Result<Vec<(Uuid, UpsertOutcome)>, CacheError> {
let mut w = begin_upsert_txn(db)?;
if durability == BatchDurability::Eventual {
w.set_durability(redb::Durability::None).map_err(cache_err)?;
}
let mut oid_index = build_oid_index(&w)?;
let mut alias_index = build_alias_index(&w)?;
let mut results = Vec::with_capacity(identities.len());
for identity in identities {
results.push(upsert_within(
&w,
identity,
namespace,
&mut OidLookup::Index(&mut oid_index),
&mut AliasLookup::Index(&mut alias_index),
)?);
}
w.commit().map_err(cache_err)?;
Ok(results)
}
fn flush_durable(db: &Database) -> Result<(), CacheError> {
let w = db.begin_write().map_err(cache_err)?;
w.commit().map_err(cache_err)?;
Ok(())
}
fn add_user_alias(db: &Database, target_id: Uuid, alias: &str) -> Result<bool, CacheError> {
let normalized = normalize(alias);
if normalized.is_empty() {
return Ok(false);
}
let tid = target_id.to_string();
let w = db.begin_write().map_err(cache_err)?;
let inserted = {
let mut table = w.open_table(ALIASES).map_err(cache_err)?;
let exists = {
let mut found = false;
for entry in table.iter().map_err(cache_err)? {
let (_, v) = entry.map_err(cache_err)?;
let sa: StoredAlias = serde_json::from_slice(v.value()).map_err(cache_err)?;
if sa.target_id == tid && sa.alias.normalized == normalized {
found = true;
break;
}
}
found
};
if exists {
false
} else {
let alias_id = Uuid::new_v4().to_string();
let sa = StoredAlias {
target_id: tid.clone(),
alias: ResolvedAlias::new(alias, AliasKind::User),
};
let bytes = serde_json::to_vec(&sa).map_err(cache_err)?;
table.insert(alias_id.as_str(), bytes.as_slice()).map_err(cache_err)?;
true
}
};
w.commit().map_err(cache_err)?;
Ok(inserted)
}
fn remove_user_alias(db: &Database, alias_id: &str) -> Result<bool, CacheError> {
let w = db.begin_write().map_err(cache_err)?;
let removed = {
let mut table = w.open_table(ALIASES).map_err(cache_err)?;
let existing = table.get(alias_id).map_err(cache_err)?.map(|v| v.value().to_vec());
match existing {
None => false,
Some(raw) => {
let sa: StoredAlias = serde_json::from_slice(&raw).map_err(cache_err)?;
if sa.alias.kind == AliasKind::User {
table.remove(alias_id).map_err(cache_err)?;
true
} else {
false
}
}
}
};
w.commit().map_err(cache_err)?;
Ok(removed)
}
fn list(db: &Database) -> Result<Vec<CachedTarget>, CacheError> {
let txn = db.begin_read().map_err(cache_err)?;
let aliases_tbl = txn.open_table(ALIASES).map_err(cache_err)?;
let mut aliases_by: HashMap<String, Vec<ResolvedAlias>> = HashMap::new();
for entry in aliases_tbl.iter().map_err(cache_err)? {
let (_, v) = entry.map_err(cache_err)?;
let sa: StoredAlias = serde_json::from_slice(v.value()).map_err(cache_err)?;
aliases_by.entry(sa.target_id).or_default().push(sa.alias);
}
let targets = txn.open_table(TARGETS).map_err(cache_err)?;
let mut out = Vec::new();
for entry in targets.iter().map_err(cache_err)? {
let (k, v) = entry.map_err(cache_err)?;
let id_str = k.value().to_string();
let st: StoredTarget = serde_json::from_slice(v.value()).map_err(cache_err)?;
let id =
Uuid::parse_str(&id_str).map_err(|e| CacheError::InvalidUuid(id_str.clone(), e))?;
let mut aliases = aliases_by.remove(&id_str).unwrap_or_default();
aliases.sort_by(|a, b| a.alias.cmp(&b.alias));
out.push(st.into_target(id, aliases));
}
out.sort_by(|a, b| a.primary_designation.cmp(&b.primary_designation));
Ok(out)
}
#[async_trait::async_trait]
impl Cache for RedbCache {
async fn get_by_id(&self, id: Uuid) -> Result<Option<CachedTarget>, CacheError> {
let db = self.db.clone();
tokio::task::spawn_blocking(move || get_by_id(&db, id)).await.map_err(cache_err)?
}
async fn get_by_simbad_oid(&self, oid: i64) -> Result<Option<CachedTarget>, CacheError> {
let db = self.db.clone();
tokio::task::spawn_blocking(move || get_by_simbad_oid(&db, oid)).await.map_err(cache_err)?
}
async fn get_by_normalized(
&self,
normalized: &str,
) -> Result<Option<CachedTarget>, CacheError> {
let db = self.db.clone();
let normalized = normalized.to_owned();
tokio::task::spawn_blocking(move || get_by_normalized(&db, &normalized))
.await
.map_err(cache_err)?
}
async fn search(&self, query: &str, limit: usize) -> Result<Vec<SearchHit>, CacheError> {
let db = self.db.clone();
let query = query.to_owned();
tokio::task::spawn_blocking(move || search(&db, &query, limit)).await.map_err(cache_err)?
}
async fn upsert(
&self,
identity: &ResolvedIdentity,
namespace: &Uuid,
) -> Result<(Uuid, UpsertOutcome), CacheError> {
let db = self.db.clone();
let identity = identity.clone();
let namespace = *namespace;
tokio::task::spawn_blocking(move || upsert(&db, &identity, &namespace))
.await
.map_err(cache_err)?
}
async fn upsert_batch(
&self,
identities: &[ResolvedIdentity],
namespace: &Uuid,
) -> Result<Vec<(Uuid, UpsertOutcome)>, CacheError> {
let db = self.db.clone();
let identities = identities.to_vec();
let namespace = *namespace;
let durability = self.bulk_durability;
tokio::task::spawn_blocking(move || upsert_batch(&db, &identities, &namespace, durability))
.await
.map_err(cache_err)?
}
async fn add_user_alias(&self, target_id: Uuid, alias: &str) -> Result<bool, CacheError> {
let db = self.db.clone();
let alias = alias.to_owned();
tokio::task::spawn_blocking(move || add_user_alias(&db, target_id, &alias))
.await
.map_err(cache_err)?
}
async fn remove_user_alias(&self, alias_id: &str) -> Result<bool, CacheError> {
let db = self.db.clone();
let alias_id = alias_id.to_owned();
tokio::task::spawn_blocking(move || remove_user_alias(&db, &alias_id))
.await
.map_err(cache_err)?
}
async fn list(&self) -> Result<Vec<CachedTarget>, CacheError> {
let db = self.db.clone();
tokio::task::spawn_blocking(move || list(&db)).await.map_err(cache_err)?
}
}
impl RedbCache {
pub async fn flush(&self) -> Result<(), CacheError> {
let db = self.db.clone();
tokio::task::spawn_blocking(move || flush_durable(&db)).await.map_err(cache_err)?
}
}
fn enqueue(db: &Database, id: &str, query: &str) -> Result<(), QueueError> {
let w = db.begin_write().map_err(queue_err)?;
{
let mut pending = w.open_table(PENDING).map_err(queue_err)?;
if pending.get(id).map_err(queue_err)?.is_none() {
let mut meta = w.open_table(META).map_err(queue_err)?;
let seq = meta.get(PENDING_SEQ_KEY).map_err(queue_err)?.map_or(0, |g| g.value());
meta.insert(PENDING_SEQ_KEY, seq + 1).map_err(queue_err)?;
let sp = StoredPending {
query: query.to_owned(),
state: PendingState::Pending.as_wire().to_owned(),
attempts: 0,
target_id: None,
seq,
};
let bytes = serde_json::to_vec(&sp).map_err(queue_err)?;
pending.insert(id, bytes.as_slice()).map_err(queue_err)?;
}
}
w.commit().map_err(queue_err)?;
Ok(())
}
fn claim_pending(db: &Database, n: usize) -> Result<Vec<PendingItem>, QueueError> {
let txn = db.begin_read().map_err(queue_err)?;
let pending = txn.open_table(PENDING).map_err(queue_err)?;
let mut rows: Vec<(String, StoredPending)> = Vec::new();
for entry in pending.iter().map_err(queue_err)? {
let (k, v) = entry.map_err(queue_err)?;
let sp: StoredPending = serde_json::from_slice(v.value()).map_err(queue_err)?;
if sp.state == PendingState::Pending.as_wire() {
rows.push((k.value().to_string(), sp));
}
}
rows.sort_by_key(|(_, sp)| sp.seq);
rows.truncate(n);
rows.into_iter().map(|(id, sp)| sp.into_item(id)).collect()
}
fn update_pending(
db: &Database,
id: &str,
mutate: impl FnOnce(&mut StoredPending),
) -> Result<(), QueueError> {
let w = db.begin_write().map_err(queue_err)?;
{
let mut pending = w.open_table(PENDING).map_err(queue_err)?;
let existing = pending.get(id).map_err(queue_err)?.map(|v| v.value().to_vec());
if let Some(raw) = existing {
let mut sp: StoredPending = serde_json::from_slice(&raw).map_err(queue_err)?;
mutate(&mut sp);
let bytes = serde_json::to_vec(&sp).map_err(queue_err)?;
pending.insert(id, bytes.as_slice()).map_err(queue_err)?;
}
}
w.commit().map_err(queue_err)?;
Ok(())
}
fn get(db: &Database, id: &str) -> Result<Option<PendingItem>, QueueError> {
let txn = db.begin_read().map_err(queue_err)?;
let pending = txn.open_table(PENDING).map_err(queue_err)?;
match pending.get(id).map_err(queue_err)? {
None => Ok(None),
Some(v) => {
let sp: StoredPending = serde_json::from_slice(v.value()).map_err(queue_err)?;
Ok(Some(sp.into_item(id.to_owned())?))
}
}
}
fn pending_count(db: &Database) -> Result<usize, QueueError> {
let txn = db.begin_read().map_err(queue_err)?;
let pending = txn.open_table(PENDING).map_err(queue_err)?;
let mut count = 0usize;
for entry in pending.iter().map_err(queue_err)? {
let (_, v) = entry.map_err(queue_err)?;
let sp: StoredPending = serde_json::from_slice(v.value()).map_err(queue_err)?;
if sp.state == PendingState::Pending.as_wire() {
count += 1;
}
}
Ok(count)
}
#[async_trait::async_trait]
impl Queue for RedbQueue {
async fn enqueue(&self, id: &str, query: &str) -> Result<(), QueueError> {
let db = self.db.clone();
let (id, query) = (id.to_owned(), query.to_owned());
tokio::task::spawn_blocking(move || enqueue(&db, &id, &query)).await.map_err(queue_err)?
}
async fn claim_pending(&self, n: usize) -> Result<Vec<PendingItem>, QueueError> {
let db = self.db.clone();
tokio::task::spawn_blocking(move || claim_pending(&db, n)).await.map_err(queue_err)?
}
async fn mark_resolved(&self, id: &str, target_id: Uuid) -> Result<(), QueueError> {
let db = self.db.clone();
let id = id.to_owned();
tokio::task::spawn_blocking(move || {
update_pending(&db, &id, |sp| {
PendingState::Resolved.as_wire().clone_into(&mut sp.state);
sp.target_id = Some(target_id.to_string());
})
})
.await
.map_err(queue_err)?
}
async fn mark_unresolved(&self, id: &str) -> Result<(), QueueError> {
let db = self.db.clone();
let id = id.to_owned();
tokio::task::spawn_blocking(move || {
update_pending(&db, &id, |sp| {
PendingState::Unresolved.as_wire().clone_into(&mut sp.state);
sp.attempts += 1;
})
})
.await
.map_err(queue_err)?
}
async fn release(&self, id: &str) -> Result<(), QueueError> {
let db = self.db.clone();
let id = id.to_owned();
tokio::task::spawn_blocking(move || {
update_pending(&db, &id, |sp| {
PendingState::Pending.as_wire().clone_into(&mut sp.state);
})
})
.await
.map_err(queue_err)?
}
async fn get(&self, id: &str) -> Result<Option<PendingItem>, QueueError> {
let db = self.db.clone();
let id = id.to_owned();
tokio::task::spawn_blocking(move || get(&db, &id)).await.map_err(queue_err)?
}
async fn pending_count(&self) -> Result<usize, QueueError> {
let db = self.db.clone();
tokio::task::spawn_blocking(move || pending_count(&db)).await.map_err(queue_err)?
}
}
#[cfg(test)]
mod tests {
use super::*;
fn m31() -> ResolvedIdentity {
ResolvedIdentity {
simbad_oid: Some(1_575_544),
primary_designation: "M 31".to_owned(),
common_name: Some("Andromeda Galaxy".to_owned()),
object_type: ObjectType::Galaxy,
otype_raw: "G".to_owned(),
ra_deg: 10.684_708,
dec_deg: 41.268_75,
v_mag: Some(3.44),
aliases: vec![
ResolvedAlias::new("M 31", AliasKind::Designation),
ResolvedAlias::new("NGC 224", AliasKind::Designation),
ResolvedAlias::new("Andromeda Galaxy", AliasKind::CommonName),
],
source: TargetSource::Resolved,
}
}
fn alias_id_of(db: &Database, display: &str) -> String {
let txn = db.begin_read().unwrap();
let table = txn.open_table(ALIASES).unwrap();
for entry in table.iter().unwrap() {
let (k, v) = entry.unwrap();
let sa: StoredAlias = serde_json::from_slice(v.value()).unwrap();
if sa.alias.alias == display {
return k.value().to_string();
}
}
panic!("alias {display:?} not found");
}
static TXN_COUNT_GUARD: std::sync::LazyLock<tokio::sync::Mutex<()>> =
std::sync::LazyLock::new(|| tokio::sync::Mutex::new(()));
#[tokio::test]
async fn remove_user_alias_only_removes_user_kind() {
let _guard = TXN_COUNT_GUARD.lock().await;
let store = Store::in_memory().unwrap();
let cache = store.cache();
let ns = crate::identity::namespace("redb.tests");
let (id, _) = cache.upsert(&m31(), &ns).await.unwrap();
cache.add_user_alias(id, "M31 favorite").await.unwrap();
let user_alias_id = alias_id_of(&store.db, "M31 favorite");
assert!(cache.remove_user_alias(&user_alias_id).await.unwrap());
assert_eq!(cache.get_by_id(id).await.unwrap().unwrap().aliases.len(), 3);
let designation_id = alias_id_of(&store.db, "M 31");
assert!(!cache.remove_user_alias(&designation_id).await.unwrap());
assert_eq!(cache.get_by_id(id).await.unwrap().unwrap().aliases.len(), 3);
}
fn distinct_identities(n: usize) -> Vec<ResolvedIdentity> {
(0..n)
.map(|i| {
let idx = u32::try_from(i).expect("test batch sizes fit in u32");
let designation = format!("Batch Target {i}");
ResolvedIdentity {
simbad_oid: Some(9_000_000 + i64::from(idx)),
primary_designation: designation.clone(),
common_name: None,
object_type: ObjectType::Galaxy,
otype_raw: "G".to_owned(),
ra_deg: f64::from(idx % 360),
dec_deg: 0.0,
v_mag: None,
aliases: vec![ResolvedAlias::new(designation, AliasKind::Designation)],
source: TargetSource::Seed,
}
})
.collect()
}
#[tokio::test]
async fn upsert_batch_uses_one_write_transaction_regardless_of_batch_size() {
let _guard = TXN_COUNT_GUARD.lock().await;
UPSERT_WRITE_TXN_COUNT.store(0, std::sync::atomic::Ordering::Relaxed);
let store = Store::in_memory().unwrap();
let cache = store.cache();
let ns = crate::identity::namespace("redb.tests.batch-txn-count");
let identities = distinct_identities(25);
let results = cache.upsert_batch(&identities, &ns).await.unwrap();
assert_eq!(results.len(), 25);
assert_eq!(
UPSERT_WRITE_TXN_COUNT.load(std::sync::atomic::Ordering::Relaxed),
1,
"one upsert_batch call of 25 identities must open exactly one write transaction"
);
UPSERT_WRITE_TXN_COUNT.store(0, std::sync::atomic::Ordering::Relaxed);
for identity in &identities {
cache.upsert(identity, &ns).await.unwrap();
}
assert_eq!(
UPSERT_WRITE_TXN_COUNT.load(std::sync::atomic::Ordering::Relaxed),
25,
"25 sequential upsert calls must open one write transaction each"
);
}
#[tokio::test]
async fn upsert_batch_does_one_full_scan_per_table_regardless_of_batch_size() {
let _guard = TXN_COUNT_GUARD.lock().await;
FULL_TABLE_SCAN_COUNT.store(0, std::sync::atomic::Ordering::Relaxed);
let store = Store::in_memory().unwrap();
let cache = store.cache();
let ns = crate::identity::namespace("redb.tests.batch-scan-count");
let identities = distinct_identities(50);
cache.upsert_batch(&identities, &ns).await.unwrap();
assert_eq!(
FULL_TABLE_SCAN_COUNT.load(std::sync::atomic::Ordering::Relaxed),
2,
"one upsert_batch call of 50 identities must do exactly one full-table oid scan \
(build_oid_index) plus one full-table alias scan (build_alias_index), total 2"
);
FULL_TABLE_SCAN_COUNT.store(0, std::sync::atomic::Ordering::Relaxed);
for identity in &identities {
cache.upsert(identity, &ns).await.unwrap();
}
assert_eq!(
FULL_TABLE_SCAN_COUNT.load(std::sync::atomic::Ordering::Relaxed),
100,
"50 sequential upsert calls must each do their own oid scan + alias scan, total 100"
);
}
#[tokio::test]
async fn upsert_batch_matches_sequential_upsert_semantics() {
let _guard = TXN_COUNT_GUARD.lock().await;
let ns = crate::identity::namespace("redb.tests.batch-semantics");
let store = Store::in_memory().unwrap();
let cache = store.cache();
let (override_id, _) = cache.upsert(&m31(), &ns).await.unwrap(); let mut user_override = m31();
user_override.source = TargetSource::UserOverride;
"My Andromeda".clone_into(&mut user_override.primary_designation);
cache.upsert(&user_override, &ns).await.unwrap();
let mut reseed = m31(); "WRONG".clone_into(&mut reseed.primary_designation);
let mut other = distinct_identities(1).remove(0);
let batch = vec![other.clone(), reseed.clone()];
let outcomes = cache.upsert_batch(&batch, &ns).await.unwrap();
assert_eq!(outcomes[1].0, override_id);
assert_eq!(outcomes[1].1, UpsertOutcome::SkippedUserOverride);
let got = cache.get_by_id(override_id).await.unwrap().unwrap();
assert_eq!(got.primary_designation, "My Andromeda", "user override stays sticky");
other.simbad_oid = None;
"Repeat Target".clone_into(&mut other.primary_designation);
let mut again = other.clone();
again.dec_deg = 12.5;
let batch2 = vec![other.clone(), again];
let outcomes2 = cache.upsert_batch(&batch2, &ns).await.unwrap();
assert_eq!(outcomes2[0].1, UpsertOutcome::Inserted, "first-ever row for this designation");
assert_eq!(outcomes2[1].1, UpsertOutcome::Updated, "dedups against entry 0 of this batch");
assert_eq!(outcomes2[0].0, outcomes2[1].0, "same derived id, same row");
let got2 = cache.get_by_id(outcomes2[1].0).await.unwrap().unwrap();
assert!((got2.dec_deg - 12.5).abs() < f64::EPSILON, "batch entry 1's value won");
}
}