use gnitz_expr::SchemaFacts;
use gnitz_wire::{WireFault, WireStatus};
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
use std::time::Duration;
use crate::client::{offload, BoxFut, DeltaPoll, GnitzClient, Host};
use crate::connection::{DeltaCursor, Polled, RawBlock, RelDescriptor, Target};
use crate::error::ClientError;
use crate::{RelName, Schema, ZSetBatch};
use gnitz_wire::txn_frame::DeltaPollItem;
use gnitz_wire::RelClass;
pub trait MirrorStore: Send {
fn base_dir(&self) -> &str;
fn register(&mut self, name: &RelName, desc: &RelDescriptor) -> Result<(), MirrorError>;
fn drop_cursor(&mut self, tid: u64);
fn forget(&mut self, tid: u64) -> Result<(), MirrorError>;
fn refill(&mut self, tid: u64) -> Result<(), MirrorError>;
fn fill(&mut self, tid: u64, blocks: &[&[u8]]) -> Result<(), MirrorError>;
fn seal(&mut self, tid: u64, cursor: DeltaCursor) -> Result<(), MirrorError>;
fn advance(&mut self, tid: u64, blocks: &[&[u8]], next: DeltaCursor) -> Result<(), MirrorError>;
fn scan_spec(
&mut self,
tid: u64,
spec: gnitz_wire::ReadSpec,
reply_schema: &Schema,
) -> Result<ZSetBatch, MirrorError>;
fn cursor_of(&self, tid: u64) -> Option<DeltaCursor>;
fn checkpoint(&mut self) -> Result<(), MirrorError>;
fn poisoned(&self) -> Option<&str>;
}
#[derive(Debug, Clone)]
pub enum MirrorError {
Engine(String),
Poisoned(String),
}
impl std::fmt::Display for MirrorError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
MirrorError::Engine(m) => write!(f, "engine: {m}"),
MirrorError::Poisoned(m) => write!(f, "mirror store is poisoned: {m}"),
}
}
}
impl std::error::Error for MirrorError {}
impl From<MirrorError> for ClientError {
fn from(e: MirrorError) -> Self {
ClientError::Mirror(e)
}
}
#[derive(Debug)]
pub struct PollOutcome {
pub view_id: u64,
pub cursor: Option<DeltaCursor>,
pub result: PollResult,
}
#[derive(Debug)]
pub enum PollResult {
Advanced,
Reseeded,
Failed(ClientError),
}
impl PollResult {
pub fn reseeded(&self) -> bool {
matches!(self, PollResult::Reseeded)
}
}
#[derive(Clone)]
pub(crate) struct MirroredView {
pub(crate) name: RelName,
pub(crate) desc: Arc<RelDescriptor>,
upstream: Target,
spec: Vec<u8>,
plan: Option<Planner>,
confirmed: bool,
}
#[derive(Clone, Debug)]
pub struct Subscription {
pub upstream: Arc<RelDescriptor>,
pub spec: Vec<u8>,
pub schema: Arc<Schema>,
pub indexes: Vec<gnitz_wire::RelIndex>,
}
pub type Planner =
Arc<dyn for<'a> Fn(&'a mut GnitzClient) -> BoxFut<'a, Result<Subscription, ClientError>> + Send + Sync>;
impl Subscription {
fn whole(upstream: Arc<RelDescriptor>) -> Self {
Subscription {
spec: gnitz_wire::ReadSpec::all_rows(gnitz_wire::ReadBound::None).encode(),
schema: Arc::clone(&upstream.schema),
indexes: upstream.indexes.clone(),
upstream,
}
}
}
fn alias_id(name: &RelName) -> u64 {
1 << 63 | gnitz_wire::checksum(name.name().as_bytes()) >> 1
}
impl MirroredView {
fn new(name: RelName, sub: Subscription, plan: Option<Planner>) -> Result<Self, ClientError> {
let Subscription { upstream, spec, schema, indexes } = sub;
let desc = match &plan {
None => {
refuse_unfed(&format!("'{name}'"), upstream.class)?;
Arc::clone(&upstream)
}
Some(_) => {
refuse_unfed(&format!("what '{name}' reads"), upstream.class)?;
Arc::new(RelDescriptor {
tid: alias_id(&name),
class: RelClass::FedView,
pk_repeats: upstream.pk_repeats,
serial: false,
schema,
indexes,
token: 0,
})
}
};
Ok(MirroredView {
name,
desc,
upstream: Target::from(&*upstream),
spec,
plan,
confirmed: false,
})
}
pub(crate) fn renamed(&self, name: RelName) -> Self {
MirroredView { name, ..self.clone() }
}
fn reads_what(&self, other: &MirroredView) -> bool {
self.upstream.tid == other.upstream.tid && self.spec == other.spec
}
fn answers_at(&self, store: &dyn MirrorStore) -> Option<DeltaCursor> {
self.confirmed.then(|| store.cursor_of(self.desc.tid)).flatten()
}
fn poll_item(&self, from: Option<DeltaCursor>) -> DeltaPollItem<'_> {
let (tag, after_tick) = DeltaCursor::flat(from);
DeltaPollItem {
view: self.upstream,
tag,
after_tick,
reply_layout: self.desc.schema.layout_digest(),
spec: &self.spec,
}
}
}
#[derive(Clone)]
pub(crate) struct Store(Arc<Mutex<dyn MirrorStore>>);
impl Store {
pub(crate) fn get(&self) -> MutexGuard<'_, dyn MirrorStore + 'static> {
self.0.lock().unwrap_or_else(PoisonError::into_inner)
}
pub(crate) async fn run<R: Send + 'static>(
self,
host: &mut dyn Host,
work: impl FnOnce(&mut dyn MirrorStore) -> R + Send + 'static,
) -> Result<R, ClientError> {
offload(host, move || work(&mut *self.get())).await
}
}
pub(crate) struct MirrorState {
pub(crate) store: Store,
pub(crate) views: HashMap<u64, MirroredView>,
owed_reseed: HashSet<u64>,
}
impl MirrorState {
fn view(&self, tid: u64) -> Result<&MirroredView, ClientError> {
self.views
.get(&tid)
.ok_or_else(|| ClientError::from(format!("relation {tid} is not mirrored")))
}
pub(crate) fn cursor_of(&self, tid: u64) -> Option<DeltaCursor> {
self.views.get(&tid)?.answers_at(&*self.store.get())
}
pub(crate) fn answering(&self, name: &RelName) -> Option<&MirroredView> {
let store = self.store.get();
self.views
.values()
.find(|v| v.name == *name && v.answers_at(&*store).is_some())
}
pub(crate) fn connection_replaced(&mut self) {
self.views.values_mut().for_each(|v| v.confirmed = false);
}
fn report(&mut self, done: Vec<(u64, Option<ClientError>)>) -> Vec<PollOutcome> {
let MirrorState { store, views, owed_reseed } = self;
let store = store.get();
done.into_iter()
.filter_map(|(view_id, failed)| {
let view = views.get(&view_id)?;
Some(PollOutcome {
view_id,
cursor: view.answers_at(&*store),
result: match failed {
Some(e) => PollResult::Failed(e),
None if owed_reseed.remove(&view_id) => PollResult::Reseeded,
None => PollResult::Advanced,
},
})
})
.collect()
}
}
const SETTLE_ATTEMPTS: usize = 4;
fn refuse_unfed(what: &str, class: RelClass) -> Result<(), ClientError> {
Err(ClientError::from(match class {
RelClass::FedView => return Ok(()),
RelClass::Table | RelClass::Stream => format!("{what} is a {}; only a view can be mirrored", class.noun()),
RelClass::BoundedView => format!(
"{what} is capacity-bounded, and a capacity and a feed \
are refused together, so it carries no feed to subscribe to"
),
RelClass::View => format!(
"{what} keeps no delta feed; \
create it WITH (delta = '<size>') to mirror it"
),
}))
}
fn no_mirror_store() -> ClientError {
ClientError::from("this client mirrors nothing; attach a store before mirroring a view".to_string())
}
impl GnitzClient {
fn mirror_state(&mut self) -> Result<&mut MirrorState, ClientError> {
self.mirror.as_deref_mut().ok_or_else(no_mirror_store)
}
fn mirrored_view(&mut self, tid: u64) -> Result<&MirroredView, ClientError> {
self.mirror_state()?.view(tid)
}
async fn derive(&mut self, name: RelName, plan: Option<Planner>) -> Result<MirroredView, ClientError> {
let sub = match &plan {
None => Subscription::whole(self.resolve_relation(&name).await?),
Some(plan) => plan(self).await?,
};
MirroredView::new(name, sub, plan)
}
pub(crate) async fn bind(&mut self, mut entry: MirroredView) -> Result<u64, ClientError> {
let tid = entry.desc.tid;
let store = self.mirror_state()?.store.clone();
let (rel, registered) = (entry.name.clone(), Arc::clone(&entry.desc));
store
.run(&mut *self.host, move |s| s.register(&rel, ®istered))
.await??;
let m = self.mirror_state()?;
let held = m.views.get(&tid);
entry.confirmed = held.is_some_and(|held| held.confirmed && held.reads_what(&entry));
m.views.retain(|_, v| v.name != entry.name);
m.views.insert(tid, entry);
Ok(tid)
}
async fn rebind(&mut self, tid: u64) -> Result<u64, ClientError> {
let v = self.mirrored_view(tid)?;
let (name, plan) = (v.name.clone(), v.plan.clone());
let entry = self.derive(name, plan).await?;
self.bind(entry).await
}
async fn mirror_as(&mut self, name: RelName, plan: Option<Planner>) -> Result<PollOutcome, ClientError> {
self.refuse_poisoned_mirror()?;
let entry = self.derive(name, plan).await?;
let tid = self.bind(entry).await?;
let done = self.settle([tid], Duration::ZERO).await?;
let out = self.mirror_state()?.report(done).pop();
let out = out.expect("the view settled is registered");
match out.result {
PollResult::Failed(e) => Err(e),
_ => Ok(out),
}
}
async fn settle(
&mut self,
tids: impl IntoIterator<Item = u64>,
mut wait: Duration,
) -> Result<Vec<(u64, Option<ClientError>)>, ClientError> {
let mut pending: Vec<u64> = tids.into_iter().collect();
let mut done = Vec::with_capacity(pending.len());
for attempt in 1..=SETTLE_ATTEMPTS {
if attempt > 1 {
let m = self.mirror_state()?;
pending.retain(|tid| m.views.contains_key(tid));
}
if pending.is_empty() {
break;
}
let synced = self.sync(&pending, wait).await?;
wait = Duration::ZERO;
pending.clear();
for (tid, result) in synced {
if attempt > 1 {
done.retain(|(synced, _)| *synced != tid);
}
let recovered = match result {
Ok(()) => {
if let Some(v) = self.mirror_state()?.views.get_mut(&tid) {
v.confirmed = true;
}
done.push((tid, None));
continue;
}
Err(e) if attempt == SETTLE_ATTEMPTS => Err(e),
Err(e) => self.recover(tid, e).await,
};
match recovered {
Ok(now) if pending.contains(&now) => {}
Ok(now) => pending.push(now),
Err(e @ ClientError::Interrupted(_)) => return Err(e),
Err(e) => done.push((tid, Some(e))),
}
}
}
Ok(done)
}
async fn recover(&mut self, tid: u64, failure: ClientError) -> Result<u64, ClientError> {
let ClientError::Refused(WireFault { status, .. }) = &failure else {
return Err(failure);
};
match status {
WireStatus::StaleCatalog => self.rebind(tid).await,
WireStatus::DeltaExpired => {
self.mirror_state()?.store.get().drop_cursor(tid);
Ok(tid)
}
_ => Err(failure),
}
}
async fn sync(&mut self, tids: &[u64], wait: Duration) -> Result<Vec<(u64, Result<(), ClientError>)>, ClientError> {
let (mut polls, mut whole) = (Vec::new(), Vec::new());
{
let store = self.mirror_state()?.store.get();
for &tid in tids {
match store.cursor_of(tid) {
Some(prev) => polls.push((tid, prev)),
None => whole.push(tid),
}
}
}
let wait = if whole.is_empty() { wait } else { Duration::ZERO };
let mut synced = self.delta_poll_many(&polls, wait).await?;
for tid in whole {
match self.bootstrap(tid).await {
Err(e @ ClientError::Interrupted(_)) => return Err(e),
read => synced.push((tid, read)),
}
}
Ok(synced)
}
async fn bootstrap(&mut self, tid: u64) -> Result<(), ClientError> {
let GnitzClient { session, host, mirror, .. } = self;
let m = mirror.as_deref().ok_or_else(no_mirror_store)?;
let view = m.view(tid)?;
let (store, item) = (m.store.clone(), view.poll_item(None));
let host = &mut **host;
let (name, desc) = (view.name.clone(), Arc::clone(&view.desc));
let open = move |s: &mut dyn MirrorStore| {
s.register(&name, &desc)?;
s.refill(tid)
};
store.clone().run(host, open).await??;
let mut poll = DeltaPoll::start(session, &[item], Duration::ZERO);
let (mut blocks, mut end) = (Vec::new(), None);
while let Some((_, polled)) = poll.next(host).await? {
match polled {
Polled::Block(b) => blocks.push(b),
Polled::End(e) => end = Some(e),
}
if poll.drained() && !blocks.is_empty() {
let blocks = std::mem::take(&mut blocks);
let fill = move |s: &mut dyn MirrorStore| {
let blocks: Vec<&[u8]> = blocks.iter().map(RawBlock::block).collect();
s.fill(tid, &blocks)
};
store.clone().run(host, fill).await??;
}
}
drop(poll);
let cursor = end.expect("one item ends once")?;
store.run(host, move |s| s.seal(tid, cursor)).await??;
self.mirror_state()?.owed_reseed.insert(tid);
Ok(())
}
}
impl GnitzClient {
pub fn attach_mirror(&mut self, store: impl MirrorStore + 'static) -> Result<(), ClientError> {
if let Some(m) = &self.mirror {
return Err(ClientError::from(format!(
"this client already mirrors at '{}'; close_mirror before attaching another store",
m.store.get().base_dir()
)));
}
self.mirror = Some(Box::new(MirrorState {
store: Store(Arc::new(Mutex::new(store))),
views: HashMap::new(),
owed_reseed: HashSet::new(),
}));
Ok(())
}
pub async fn mirror_view(&mut self, name: &RelName) -> Result<PollOutcome, ClientError> {
self.mirror_as(name.clone(), None).await
}
pub async fn mirror_subscription(&mut self, alias: &str, plan: Planner) -> Result<PollOutcome, ClientError> {
self.mirror_as(RelName::new(gnitz_wire::LOCAL_SCHEMA, alias)?, Some(plan))
.await
}
pub async fn forget_view(&mut self, table_id: u64) -> Result<(), ClientError> {
let m = self.mirror_state()?;
m.views.remove(&table_id);
let store = m.store.clone();
store.run(&mut *self.host, move |s| s.forget(table_id)).await??;
Ok(())
}
async fn delta_poll_many(
&mut self,
views: &[(u64, DeltaCursor)],
wait: Duration,
) -> Result<Vec<(u64, Result<(), ClientError>)>, ClientError> {
let GnitzClient { session, host, mirror, .. } = self;
let m = mirror.as_deref().ok_or_else(no_mirror_store)?;
let store = m.store.clone();
let host = &mut **host;
let items: Result<Vec<DeltaPollItem>, ClientError> = views
.iter()
.map(|&(tid, prev)| Ok(m.view(tid)?.poll_item(Some(prev))))
.collect();
let mut poll = DeltaPoll::start(session, &items?, wait);
let mut applied = Vec::with_capacity(views.len());
let mut blocks = Vec::new();
let mut due: Vec<(u64, Vec<RawBlock>, DeltaCursor)> = Vec::new();
while let Some((i, polled)) = poll.next(host).await? {
match polled {
Polled::Block(b) => blocks.push(b),
Polled::End(end) => {
let tid = views[i].0;
let blocks = std::mem::take(&mut blocks);
match end {
Ok(next) => due.push((tid, blocks, next)),
Err(e) => applied.push((tid, Err(e))),
}
}
}
if poll.drained() && !due.is_empty() {
let trains = std::mem::take(&mut due);
let idle = trains.iter().all(|(_, blocks, _)| blocks.is_empty());
let apply = move |s: &mut dyn MirrorStore| {
let advance = |(tid, blocks, next): (u64, Vec<RawBlock>, DeltaCursor)| {
let blocks: Vec<&[u8]> = blocks.iter().map(RawBlock::block).collect();
(tid, s.advance(tid, &blocks, next).map_err(ClientError::from))
};
trains.into_iter().map(advance).collect::<Vec<_>>()
};
applied.extend(match idle {
true => apply(&mut *store.get()),
false => store.clone().run(host, apply).await?,
});
}
}
Ok(applied)
}
pub async fn poll_mirror(&mut self, wait: Duration) -> Result<Vec<PollOutcome>, ClientError> {
self.refuse_poisoned_mirror()?;
let done = self.settle(self.mirrored_ids(), wait).await?;
Ok(self.mirror_state()?.report(done))
}
pub fn mirrored_ids(&self) -> Vec<u64> {
self.mirror
.as_deref()
.map_or_else(Vec::new, |m| m.views.keys().copied().collect())
}
pub fn mirrors(&self, table_id: u64) -> bool {
self.cursor_of(table_id).is_some()
}
pub fn cursor_of(&self, table_id: u64) -> Option<DeltaCursor> {
self.mirror.as_deref()?.cursor_of(table_id)
}
pub async fn checkpoint_mirror(&mut self) -> Result<(), ClientError> {
let store = self.mirror_state()?.store.clone();
Ok(store.run(&mut *self.host, |s| s.checkpoint()).await??)
}
pub async fn close_mirror(&mut self) -> Result<(), ClientError> {
let Some(m) = self.mirror.take() else {
return Ok(());
};
match m.store.run(&mut *self.host, |s| s.checkpoint()).await? {
Err(MirrorError::Poisoned(_)) => Ok(()),
other => other.map_err(ClientError::from),
}
}
pub fn mirror_poisoned(&self) -> Option<String> {
self.mirror
.as_deref()
.and_then(|m| m.store.get().poisoned().map(str::to_owned))
}
fn refuse_poisoned_mirror(&mut self) -> Result<(), ClientError> {
match self.mirror_state()?.store.get().poisoned() {
Some(why) => Err(MirrorError::Poisoned(why.to_string()).into()),
None => Ok(()),
}
}
}
#[cfg(test)]
#[path = "tests/mirror.rs"]
mod tests;