use gnitz_expr::SchemaFacts;
use gnitz_wire::{WireFault, WireStatus};
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
use std::time::Duration;
use crate::client::{offload, DeltaPoll, GnitzClient, Host};
use crate::connection::{DeltaCursor, Polled, RawBlock, RelDescriptor};
use crate::error::ClientError;
use crate::{RelName, Schema, ZSetBatch};
use gnitz_wire::txn_frame::DeltaPollItem;
use gnitz_wire::RelClass;
pub trait MirrorStore: Send {
fn base_dir(&self) -> &str;
fn register(&mut self, name: &RelName, desc: &RelDescriptor) -> Result<Option<u64>, MirrorError>;
fn invalidate(&mut self, tid: u64, level: Invalidate) -> Result<(), MirrorError>;
fn refill(&mut self, tid: u64) -> Result<(), MirrorError>;
fn fill(&mut self, tid: u64, blocks: &[&[u8]]) -> Result<(), MirrorError>;
fn seal(&mut self, tid: u64, cursor: DeltaCursor) -> Result<(), MirrorError>;
fn advance(&mut self, tid: u64, blocks: &[&[u8]], next: DeltaCursor) -> Result<(), MirrorError>;
fn scan_spec(
&mut self,
tid: u64,
spec: gnitz_wire::ReadSpec,
reply_schema: &Schema,
) -> Result<ZSetBatch, MirrorError>;
fn cursor_of(&self, tid: u64) -> Option<DeltaCursor>;
fn clear_cursors(&mut self);
fn checkpoint(&mut self) -> Result<(), MirrorError>;
fn poisoned(&self) -> Option<&str>;
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Invalidate {
Cursor,
Registration,
}
#[derive(Debug, Clone)]
pub enum MirrorError {
Engine(String),
Poisoned(String),
}
impl std::fmt::Display for MirrorError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
MirrorError::Engine(m) => write!(f, "engine: {m}"),
MirrorError::Poisoned(m) => write!(f, "mirror store is poisoned: {m}"),
}
}
}
impl std::error::Error for MirrorError {}
impl From<MirrorError> for ClientError {
fn from(e: MirrorError) -> Self {
ClientError::Mirror(e)
}
}
#[derive(Debug)]
pub struct PollOutcome {
pub view_id: u64,
pub cursor: Option<DeltaCursor>,
pub result: PollResult,
}
#[derive(Debug)]
pub enum PollResult {
Advanced,
Reseeded,
Failed(ClientError),
}
impl PollResult {
pub fn reseeded(&self) -> bool {
matches!(self, PollResult::Reseeded)
}
}
pub(crate) struct MirroredView {
pub(crate) name: RelName,
pub(crate) desc: Arc<RelDescriptor>,
}
impl MirroredView {
fn poll_item(&self, after_tick: u64) -> DeltaPollItem {
DeltaPollItem {
view_id: self.desc.tid,
after_tick,
reply_layout: self.desc.schema.layout_digest(),
}
}
}
#[derive(Clone)]
pub(crate) struct Store(Arc<Mutex<dyn MirrorStore>>);
impl Store {
pub(crate) fn get(&self) -> MutexGuard<'_, dyn MirrorStore + 'static> {
self.0.lock().unwrap_or_else(PoisonError::into_inner)
}
pub(crate) async fn run<R: Send + 'static>(
self,
host: &mut dyn Host,
work: impl FnOnce(&mut dyn MirrorStore) -> R + Send + 'static,
) -> Result<R, ClientError> {
offload(host, move || work(&mut *self.get())).await
}
}
pub(crate) struct MirrorState {
pub(crate) store: Store,
pub(crate) views: HashMap<u64, MirroredView>,
owed_reseed: HashSet<u64>,
}
impl MirrorState {
pub(crate) fn cursor_of(&self, tid: u64) -> Option<DeltaCursor> {
self.views.get(&tid)?;
self.store.get().cursor_of(tid)
}
}
type Train = (u64, DeltaCursor, Vec<RawBlock>, DeltaCursor);
fn advance_from(store: &mut dyn MirrorStore, (tid, prev, blocks, at): Train) -> Result<PollResult, ClientError> {
let next = prev.advanced_to(at)?;
let blocks: Vec<&[u8]> = blocks.iter().map(RawBlock::block).collect();
store.advance(tid, &blocks, next)?;
Ok(PollResult::Advanced)
}
enum Settle {
Sync,
Recover(ClientError),
Reseed,
}
type ViewPollResults = Vec<(u64, Result<PollResult, ClientError>)>;
fn no_mirror_store() -> ClientError {
ClientError::from("this client mirrors nothing; attach a store before mirroring a view".to_string())
}
impl GnitzClient {
fn mirror_state(&mut self) -> Result<&mut MirrorState, ClientError> {
self.mirror.as_deref_mut().ok_or_else(no_mirror_store)
}
fn mirrored_name(&mut self, tid: u64) -> Result<RelName, ClientError> {
Ok(self.mirrored_view(tid)?.name.clone())
}
fn mirrored_view(&mut self, tid: u64) -> Result<&MirroredView, ClientError> {
self.mirror_state()?
.views
.get(&tid)
.ok_or_else(|| ClientError::from(format!("relation {tid} is not mirrored")))
}
async fn reconcile_registration(&mut self, name: &RelName) -> Result<u64, ClientError> {
let rel = self.resolve_relation(name).await?;
match rel.class {
RelClass::FedView => self.bind(name.clone(), rel).await,
RelClass::Table | RelClass::Stream => Err(ClientError::from(format!(
"'{name}' is a {}; only a view can be mirrored",
rel.class.noun()
))),
RelClass::BoundedView => Err(ClientError::from(format!(
"view '{name}' is capacity-bounded, and a capacity and a feed \
are refused together, so it carries no feed to subscribe to"
))),
RelClass::View => Err(ClientError::from(format!(
"view '{name}' keeps no delta feed; \
create it WITH (delta = '<size>') to mirror it"
))),
}
}
pub(crate) async fn bind(&mut self, name: RelName, desc: Arc<RelDescriptor>) -> Result<u64, ClientError> {
let tid = desc.tid;
let store = self.mirror_state()?.store.clone();
let (rel, registered) = (name.clone(), Arc::clone(&desc));
let register = move |s: &mut dyn MirrorStore| s.register(&rel, ®istered);
let retracted = store.run(&mut *self.host, register).await??;
let entry = MirroredView { name, desc };
let m = self.mirror_state()?;
if let Some(old) = retracted {
m.views.remove(&old);
}
m.views.insert(tid, entry);
Ok(tid)
}
async fn settle(&mut self, mut tid: u64, mut step: Settle) -> Result<(u64, PollResult), ClientError> {
loop {
step = match step {
Settle::Sync => {
let Some(prev) = self.mirror_state()?.store.get().cursor_of(tid) else {
return self.bootstrap(tid).await;
};
let item = self.mirrored_view(tid)?.poll_item(prev.tick.get());
let (_, polled) = self
.delta_poll_many(&[(prev, item)], Duration::ZERO)
.await?
.pop()
.expect("one view, one result");
match polled {
Ok(result) => return Ok((tid, result)),
Err(e) => Settle::Recover(e),
}
}
Settle::Recover(err) => match err {
ClientError::Refused(WireFault { status: WireStatus::DeltaExpired, .. }) => Settle::Reseed,
e @ ClientError::Refused(WireFault { status: WireStatus::NotFound, .. }) => {
match self.relation_id_moved(tid).await {
Ok(true) => Settle::Reseed,
Err(i @ ClientError::Interrupted(_)) => return Err(i),
Ok(false) | Err(_) => return Err(e),
}
}
e => return Err(e),
},
Settle::Reseed => {
self.mirror_state()?.store.get().invalidate(tid, Invalidate::Cursor)?;
let name = self.mirrored_name(tid)?;
tid = self.reconcile_registration(&name).await?;
Settle::Sync
}
}
}
}
async fn relation_id_moved(&mut self, tid: u64) -> Result<bool, ClientError> {
let name = self.mirrored_name(tid)?;
Ok(self.resolve_relation(&name).await?.tid != tid)
}
async fn bootstrap(&mut self, tid: u64) -> Result<(u64, PollResult), ClientError> {
let item = self.mirrored_view(tid)?.poll_item(0);
let store = self.mirror_state()?.store.clone();
let GnitzClient { session, host, .. } = self;
let host = &mut **host;
store.clone().run(host, move |s| s.refill(tid)).await??;
let mut poll = DeltaPoll::start(session, &[item], Duration::ZERO);
let (mut blocks, mut end, mut refused) = (Vec::new(), None, None);
while let Some((_, polled)) = poll.next(host).await? {
match polled {
Polled::Block(b) => blocks.push(b),
Polled::End(e) => end = Some(e),
}
if poll.drained() && !blocks.is_empty() {
let blocks = std::mem::take(&mut blocks);
if refused.is_none() {
let fill = move |s: &mut dyn MirrorStore| {
let blocks: Vec<&[u8]> = blocks.iter().map(RawBlock::block).collect();
s.fill(tid, &blocks)
};
refused = store.clone().run(host, fill).await?.err();
}
}
}
drop(poll);
if let Some(e) = refused {
return Err(e.into());
}
let cursor = end.expect("one item ends once")?;
store.run(host, move |s| s.seal(tid, cursor)).await??;
self.mirror_state()?.owed_reseed.insert(tid);
Ok((tid, PollResult::Reseeded))
}
fn outcome(&mut self, view_id: u64, result: PollResult) -> PollOutcome {
let owed = self.mirror.as_deref().is_some_and(|m| m.owed_reseed.contains(&view_id));
let result = match (result, owed) {
(PollResult::Advanced, true) => PollResult::Reseeded,
(r, _) => r,
};
PollOutcome {
view_id,
cursor: self.cursor_of(view_id),
result,
}
}
fn reported(&mut self, out: &[PollOutcome]) {
if let Some(m) = self.mirror.as_deref_mut() {
for o in out.iter().filter(|o| o.result.reseeded()) {
m.owed_reseed.remove(&o.view_id);
}
}
}
fn record(&mut self, out: &mut Vec<PollOutcome>, view_id: u64, result: PollResult) {
let o = self.outcome(view_id, result);
let Some(prev) = out.iter_mut().find(|p| p.view_id == o.view_id) else {
out.push(o);
return;
};
if o.cursor.map(|c| c.tick) > prev.cursor.map(|c| c.tick) {
prev.cursor = o.cursor;
}
if o.result.reseeded() {
prev.result = PollResult::Reseeded;
}
}
}
impl GnitzClient {
pub fn attach_mirror(&mut self, store: impl MirrorStore + 'static) -> Result<(), ClientError> {
if let Some(m) = &self.mirror {
return Err(ClientError::from(format!(
"this client already mirrors at '{}'; close_mirror before attaching another store",
m.store.get().base_dir()
)));
}
self.mirror = Some(Box::new(MirrorState {
store: Store(Arc::new(Mutex::new(store))),
views: HashMap::new(),
owed_reseed: HashSet::new(),
}));
Ok(())
}
pub async fn mirror_view(&mut self, name: &RelName) -> Result<PollOutcome, ClientError> {
self.refuse_poisoned_mirror()?;
let tid = self.reconcile_registration(name).await?;
let (id, result) = self.settle(tid, Settle::Sync).await?;
let out = self.outcome(id, result);
self.reported(std::slice::from_ref(&out));
Ok(out)
}
pub async fn forget_view(&mut self, table_id: u64) -> Result<(), ClientError> {
let m = self.mirror_state()?;
m.views.remove(&table_id);
let store = m.store.clone();
store
.run(&mut *self.host, move |s| {
s.invalidate(table_id, Invalidate::Registration)
})
.await??;
Ok(())
}
async fn delta_poll_many(
&mut self,
views: &[(DeltaCursor, DeltaPollItem)],
wait: Duration,
) -> Result<ViewPollResults, ClientError> {
let store = self.mirror_state()?.store.clone();
let GnitzClient { session, host, .. } = self;
let host = &mut **host;
let items: Vec<DeltaPollItem> = views.iter().map(|&(_, item)| item).collect();
let mut poll = DeltaPoll::start(session, &items, wait);
let mut applied = Vec::with_capacity(views.len());
let mut blocks = Vec::new();
let mut due: Vec<Train> = Vec::new();
while let Some((i, polled)) = poll.next(host).await? {
match polled {
Polled::Block(b) => blocks.push(b),
Polled::End(end) => {
let (prev, DeltaPollItem { view_id: tid, .. }) = views[i];
let blocks = std::mem::take(&mut blocks);
match end {
Ok(at) => due.push((tid, prev, blocks, at)),
Err(e) => applied.push((tid, Err(e))),
}
}
}
if poll.drained() && !due.is_empty() {
let trains = std::mem::take(&mut due);
let idle = trains.iter().all(|(_, _, blocks, _)| blocks.is_empty());
let apply = move |s: &mut dyn MirrorStore| {
let apply = |train: Train| (train.0, advance_from(s, train));
trains.into_iter().map(apply).collect::<Vec<_>>()
};
applied.extend(match idle {
true => apply(&mut *store.get()),
false => store.clone().run(host, apply).await?,
});
}
}
Ok(applied)
}
pub async fn poll_mirror(&mut self, wait: Duration) -> Result<Vec<PollOutcome>, ClientError> {
self.refuse_poisoned_mirror()?;
let mut requests: Vec<(DeltaCursor, DeltaPollItem)> = Vec::new();
let mut recoveries: Vec<(u64, Option<ClientError>)> = Vec::new();
{
let m = self.mirror_state()?;
let store = m.store.get();
for (&tid, v) in m.views.iter() {
match store.cursor_of(tid) {
Some(prev) => requests.push((prev, v.poll_item(prev.tick.get()))),
None => recoveries.push((tid, None)),
}
}
}
let wait = if recoveries.is_empty() { wait } else { Duration::ZERO };
let applied = self.delta_poll_many(&requests, wait).await?;
let mut out: Vec<PollOutcome> = Vec::with_capacity(applied.len() + recoveries.len());
for (tid, r) in applied {
match r {
Ok(result) => self.record(&mut out, tid, result),
Err(e) => recoveries.push((tid, Some(e))),
}
}
for (tid, err) in recoveries {
if !self.mirror_state()?.views.contains_key(&tid) {
continue;
}
let step = err.map_or(Settle::Reseed, Settle::Recover);
match self.settle(tid, step).await {
Ok((id, result)) => self.record(&mut out, id, result),
Err(e @ ClientError::Interrupted(_)) => return Err(e),
Err(e) => self.record(&mut out, tid, PollResult::Failed(e)),
}
}
self.reported(&out);
Ok(out)
}
pub fn mirrored_ids(&self) -> Vec<u64> {
self.mirror
.as_deref()
.map_or_else(Vec::new, |m| m.views.keys().copied().collect())
}
pub fn mirrors(&self, table_id: u64) -> bool {
self.cursor_of(table_id).is_some()
}
pub fn cursor_of(&self, table_id: u64) -> Option<DeltaCursor> {
self.mirror.as_deref()?.cursor_of(table_id)
}
pub async fn checkpoint_mirror(&mut self) -> Result<(), ClientError> {
let store = self.mirror_state()?.store.clone();
Ok(store.run(&mut *self.host, |s| s.checkpoint()).await??)
}
pub async fn close_mirror(&mut self) -> Result<(), ClientError> {
let Some(m) = self.mirror.take() else {
return Ok(());
};
match m.store.run(&mut *self.host, |s| s.checkpoint()).await? {
Err(MirrorError::Poisoned(_)) => Ok(()),
other => other.map_err(ClientError::from),
}
}
pub fn mirror_poisoned(&self) -> Option<String> {
self.mirror
.as_deref()
.and_then(|m| m.store.get().poisoned().map(str::to_owned))
}
fn refuse_poisoned_mirror(&mut self) -> Result<(), ClientError> {
match self.mirror_state()?.store.get().poisoned() {
Some(why) => Err(MirrorError::Poisoned(why.to_string()).into()),
None => Ok(()),
}
}
}
#[cfg(test)]
#[path = "tests/mirror.rs"]
mod tests;