artifactize_core/

changes.rs

//! Best-effort, state-scoped invalidation hints. SQLite and session files remain authoritative.
//! The first subscriber temporarily hosts a hub; there is no daemon and writers never elect one.
mod probe;
#[cfg(test)]
pub(crate) mod test_support;
#[cfg(test)]
mod tests;
mod transport;

use crate::types::{RunId, SessionId};
use serde::{Deserialize, Serialize};
use std::{
    collections::{BTreeMap, BTreeSet},
    path::Path,
    sync::{Arc, Mutex, OnceLock, Weak},
    time::Duration,
};
use tokio::{
    io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt},
    sync::{Notify, broadcast, oneshot},
    task::JoinSet,
};
use tokio_util::sync::CancellationToken;
use transport::Endpoint;

/// Reconcile missed commits and database replacement without reloading idle snapshots.
pub const RECONCILE: Duration = Duration::from_secs(5);
/// A dead hub is replaced promptly, without a spin loop or a permanent process.
const RECONNECT: Duration = Duration::from_millis(500);
/// Bound startup and short-command shutdown; notification availability cannot block correctness.
const DELIVERY_TIMEOUT: Duration = Duration::from_millis(300);
/// Bound hostile frames before allocation (IDs are at most 200 bytes).
const MAX_FRAME_BYTES: usize = 1024;
/// Bound per-process publishers, per-hub connections, broadcast backlog and dirty sets.
const MAX_STATES: usize = 32;
const MAX_CLIENTS: usize = 64;
const MAX_BACKLOG: usize = 64;
const MAX_DIRTY_SESSIONS: usize = 64;
const MAX_DIRTY_RUNS: usize = 64;
/// Version 2 names the Run a write changed ([`Change::RunInvalidated`]). The wire format is
/// independent of the state database schema and durable session-event format. The Hello
/// handshake rejects any unsupported version instead of interpreting incompatible hints; a
/// reader that cannot connect reconciles every [`RECONCILE`] instead.
const VERSION: u32 = 2;

/// Hints carry no records, credentials, scheduler decisions or remote-store events.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum Change {
    /// The list of Runs changed: a Run was created or removed, or a write could not be tied to
    /// a Run. Any Run may have changed with it.
    StateInvalidated,
    /// That Run's records changed: its state, a request, or an execution one of its requests
    /// holds.
    RunInvalidated(RunId),
    SessionInvalidated(SessionId),
    /// Initial registration, reconnect, gap, overflow or database replacement: read afresh.
    Resync,
}

/// What the hints since a reader's last read say it must read again: nothing, these Runs, or
/// everything. Session hints are not about the state database and change nothing here.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub enum Changed {
    #[default]
    Nothing,
    Runs(BTreeSet<RunId>),
    Everything,
}
impl Changed {
    pub fn add(&mut self, change: &Change) {
        match change {
            Change::StateInvalidated | Change::Resync => *self = Self::Everything,
            Change::RunInvalidated(id) => match self {
                Self::Nothing => *self = Self::Runs(BTreeSet::from([id.clone()])),
                Self::Runs(ids) => {
                    ids.insert(id.clone());
                    if ids.len() > MAX_DIRTY_RUNS {
                        *self = Self::Everything;
                    }
                }
                Self::Everything => {}
            },
            Change::SessionInvalidated(_) => {}
        }
    }
    /// Keep what `other` says changed as well, as when a read that took it was not used.
    pub fn merge(&mut self, other: Self) {
        match other {
            Self::Nothing => {}
            Self::Everything => *self = Self::Everything,
            Self::Runs(ids) => {
                for id in ids {
                    self.add(&Change::RunInvalidated(id));
                }
            }
        }
    }
    /// What a read reads again: what the hints named or, when the database was `written` and
    /// no hint named the change yet (a writer that could not reach the hub), everything.
    pub fn or_written(&self, written: bool) -> Self {
        match self {
            Self::Nothing if written => Self::Everything,
            changed => changed.clone(),
        }
    }
    /// Whether Run `id` may have changed.
    pub fn run(&self, id: &RunId) -> bool {
        match self {
            Self::Nothing => false,
            Self::Runs(ids) => ids.contains(id),
            Self::Everything => true,
        }
    }
}

/// Changes not yet delivered: a resync replaces everything else pending, and a state change
/// replaces pending Run changes.
enum Dirty {
    Pending {
        state: bool,
        runs: BTreeSet<RunId>,
        sessions: BTreeSet<SessionId>,
    },
    Resync,
}
impl Default for Dirty {
    fn default() -> Self {
        Self::Pending {
            state: false,
            runs: BTreeSet::new(),
            sessions: BTreeSet::new(),
        }
    }
}
impl Dirty {
    fn add(&mut self, change: Change) {
        let Self::Pending {
            state,
            runs,
            sessions,
        } = self
        else {
            return;
        };
        match change {
            Change::Resync => *self = Self::Resync,
            Change::StateInvalidated => {
                *state = true;
                runs.clear();
            }
            Change::RunInvalidated(id) => {
                if !*state {
                    runs.insert(id);
                    if runs.len() > MAX_DIRTY_RUNS {
                        *state = true;
                        runs.clear();
                    }
                }
            }
            Change::SessionInvalidated(id) => {
                sessions.insert(id);
                if sessions.len() > MAX_DIRTY_SESSIONS {
                    *self = Self::Resync;
                }
            }
        }
    }
    fn pop(&mut self) -> Option<Change> {
        match self {
            Self::Resync => {
                *self = Self::default();
                Some(Change::Resync)
            }
            Self::Pending {
                state,
                runs,
                sessions,
            } => {
                if std::mem::take(state) {
                    Some(Change::StateInvalidated)
                } else if let Some(id) = runs.pop_first() {
                    Some(Change::RunInvalidated(id))
                } else {
                    sessions.pop_first().map(Change::SessionInvalidated)
                }
            }
        }
    }
    fn is_empty(&self) -> bool {
        matches!(self, Self::Pending { state: false, runs, sessions } if runs.is_empty() && sessions.is_empty())
    }
}
struct Inbox {
    #[cfg(test)]
    registration: tokio::sync::watch::Sender<Option<crate::config::HubEpoch>>,
    // A test can wait for received hints rather than confusing the hub's ACK with delivery.
    #[cfg(test)]
    received: tokio::sync::watch::Sender<Option<u64>>,
    dirty: Mutex<Dirty>,
    wake: Notify,
}
impl Default for Inbox {
    fn default() -> Self {
        Self {
            #[cfg(test)]
            registration: tokio::sync::watch::channel(None).0,
            #[cfg(test)]
            received: tokio::sync::watch::channel(None).0,
            dirty: Mutex::default(),
            wake: Notify::new(),
        }
    }
}
impl Inbox {
    fn add(&self, change: Change) {
        self.dirty.lock().unwrap().add(change);
        self.wake.notify_one();
    }
    async fn next(&self) -> Change {
        loop {
            // Register before checking: a hint arriving between check and await is retained.
            let wake = self.wake.notified();
            if let Some(change) = self.dirty.lock().unwrap().pop() {
                return change;
            }
            wake.await;
        }
    }
}

/// Register before reading the baseline. Dirties arriving during that read stay queued.
/// Drop requests cancellation; await [`Self::close`] before removing the subscribed state.
pub struct Subscription {
    inbox: Arc<Inbox>,
    cancel: CancellationToken,
    task: tokio::task::JoinHandle<Result<(), String>>,
}
impl Subscription {
    pub async fn new(state: &Path) -> Self {
        let inbox = Arc::new(Inbox::default());
        let cancel = CancellationToken::new();
        let (ready, registered) = oneshot::channel();
        let state = state.to_path_buf();
        let task = tokio::spawn(watch(state, inbox.clone(), cancel.clone(), ready));
        // Keep cancellation ownership even if the caller abandons registration midway.
        let subscription = Self {
            inbox,
            cancel,
            task,
        };
        // An unavailable or unsafe endpoint degrades, never prevents a baseline read.
        let _ = registered.await;
        subscription
    }
    /// Stop this subscription and wait for its connection, hub and probe database to close.
    /// Other subscriptions remain independent. Dropping this future still requests cancellation.
    pub async fn close(mut self) -> Result<(), String> {
        self.cancel.cancel();
        crate::task::joined((&mut self.task).await).map_err(|error| error.to_string())?
    }
    /// Process-owner death has no database writer; reuse the subscription's reconciliation
    /// cadence when an observer also owns liveness reconciliation.
    pub async fn next_reconciled(&mut self) -> Change {
        tokio::time::timeout(RECONCILE, self.next_state())
            .await
            .unwrap_or(Change::Resync)
    }
    pub async fn next(&mut self) -> Change {
        self.inbox.next().await
    }
    /// Schedulers and non-session UIs can ignore session-only hints without reloading state.
    /// A Run's hint is returned: the caller reads that Run again, or the state.
    pub async fn next_state(&mut self) -> Change {
        loop {
            let change = self.next().await;
            if !matches!(change, Change::SessionInvalidated(_)) {
                return change;
            }
        }
    }
}
impl Drop for Subscription {
    fn drop(&mut self) {
        self.cancel.cancel();
    }
}

#[derive(Default)]
struct Outbox {
    dirty: Dirty,
    submitted: u64,
    delivered: u64,
}
struct Publishing {
    endpoint: Endpoint,
    pending: Mutex<Outbox>,
    wake: Notify,
    drained: Notify,
    // Fixture barriers must distinguish an actual hub ACK from best-effort timeout.
    #[cfg(test)]
    delivery_error: Mutex<Option<String>>,
}
/// Cloneable synchronous producer, also safe to invoke on the SQLite worker thread.
#[derive(Clone, Default)]
pub(crate) struct Publisher(Option<Arc<Publishing>>);
static PUBLISHERS: OnceLock<Mutex<BTreeMap<crate::config::EndpointId, Weak<Publishing>>>> =
    OnceLock::new();
impl Publisher {
    pub(crate) fn new(state: &Path) -> Self {
        let Ok(runtime) = tokio::runtime::Handle::try_current() else {
            return Self::default();
        };
        let Ok(endpoint) = Endpoint::new(state) else {
            return Self::default();
        };
        let mut publishers = PUBLISHERS.get_or_init(Default::default).lock().unwrap();
        publishers.retain(|_, publisher| publisher.strong_count() != 0);
        if let Some(publisher) = publishers.get(&endpoint.identity).and_then(Weak::upgrade) {
            return Self(Some(publisher));
        }
        if publishers.len() >= MAX_STATES {
            return Self::default();
        }
        let publisher = Arc::new(Publishing {
            endpoint,
            pending: Mutex::new(Outbox::default()),
            wake: Notify::new(),
            drained: Notify::new(),
            #[cfg(test)]
            delivery_error: Mutex::default(),
        });
        publishers.insert(
            publisher.endpoint.identity.clone(),
            Arc::downgrade(&publisher),
        );
        runtime.spawn(publish(publisher.clone()));
        Self(Some(publisher))
    }
    pub(crate) fn notify(&self, change: Change) {
        if let Some(publisher) = &self.0 {
            let mut pending = publisher.pending.lock().unwrap();
            pending.dirty.add(change);
            pending.submitted = pending.submitted.wrapping_add(1);
            drop(pending);
            publisher.wake.notify_one();
        }
    }
}

/// Drain short-lived commands centrally. Only bounded hints are sent; a failed delivery is safe.
pub async fn drain() {
    let publishers: Vec<_> = PUBLISHERS
        .get()
        .map(|publishers| {
            publishers
                .lock()
                .unwrap()
                .values()
                .filter_map(Weak::upgrade)
                .collect()
        })
        .unwrap_or_default();
    let wait = async {
        for publisher in publishers {
            let target = publisher.pending.lock().unwrap().submitted;
            loop {
                let drained = publisher.drained.notified();
                if publisher.pending.lock().unwrap().delivered >= target {
                    break;
                }
                drained.await;
            }
        }
    };
    let _ = tokio::time::timeout(DELIVERY_TIMEOUT, wait).await;
}

#[derive(Serialize, Deserialize)]
enum Frame {
    Hello {
        version: u32,
        identity: crate::config::EndpointId,
        subscriber: bool,
    },
    Registered {
        epoch: crate::config::HubEpoch,
    },
    Publish(Change),
    Delivered,
    Hint {
        epoch: crate::config::HubEpoch,
        sequence: u64,
        change: Change,
    },
}
async fn read_frame<R: AsyncRead + Unpin>(reader: &mut R) -> std::io::Result<Frame> {
    let bytes = reader.read_u32().await? as usize;
    if bytes == 0 || bytes > MAX_FRAME_BYTES {
        return Err(std::io::ErrorKind::InvalidData.into());
    }
    let mut data = vec![0; bytes];
    reader.read_exact(&mut data).await?;
    serde_json::from_slice(&data).map_err(std::io::Error::other)
}
async fn write_frame<W: AsyncWrite + Unpin>(writer: &mut W, frame: &Frame) -> std::io::Result<()> {
    let data = serde_json::to_vec(frame).map_err(std::io::Error::other)?;
    if data.len() > MAX_FRAME_BYTES {
        return Err(std::io::ErrorKind::InvalidData.into());
    }
    writer.write_u32(data.len() as u32).await?;
    writer.write_all(&data).await?;
    writer.flush().await
}
async fn connect(
    endpoint: &Endpoint,
    subscriber: bool,
) -> std::io::Result<(transport::Stream, crate::config::HubEpoch)> {
    let mut stream = endpoint.connect().await?;
    write_frame(
        &mut stream,
        &Frame::Hello {
            version: VERSION,
            identity: endpoint.identity.clone(),
            subscriber,
        },
    )
    .await?;
    match read_frame(&mut stream).await? {
        Frame::Registered { epoch } if epoch.len() <= crate::types::MAX_ID_BYTES => {
            Ok((stream, epoch))
        }
        _ => Err(std::io::ErrorKind::InvalidData.into()),
    }
}
async fn publish(publisher: Arc<Publishing>) {
    loop {
        let wake = publisher.wake.notified();
        if publisher.pending.lock().unwrap().dirty.is_empty() {
            tokio::select! {
                _ = wake => {},
                _ = tokio::time::sleep(RECONNECT) => {
                    // The task must not retain a publisher forever after its writers disappear.
                    if Arc::strong_count(&publisher) == 1 {
                        break;
                    }
                    continue;
                }
            }
        }
        let (mut dirty, target) = {
            let mut pending = publisher.pending.lock().unwrap();
            (std::mem::take(&mut pending.dirty), pending.submitted)
        };
        // No retries when no reader exists: future registration and the probe recover all state.
        let delivery = async {
            let (mut stream, _) = connect(&publisher.endpoint, false).await?;
            while let Some(change) = dirty.pop() {
                write_frame(&mut stream, &Frame::Publish(change)).await?;
                if !matches!(read_frame(&mut stream).await?, Frame::Delivered) {
                    return Err(std::io::ErrorKind::InvalidData.into());
                }
            }
            Ok::<_, std::io::Error>(())
        };
        let _delivery = tokio::time::timeout(DELIVERY_TIMEOUT, delivery).await;
        #[cfg(test)]
        if !matches!(&_delivery, Ok(Ok(()))) {
            *publisher.delivery_error.lock().unwrap() = Some(format!("{_delivery:?}"));
        }
        publisher.pending.lock().unwrap().delivered = target;
        publisher.drained.notify_one();
    }
}

async fn watch(
    state: std::path::PathBuf,
    inbox: Arc<Inbox>,
    cancel: CancellationToken,
    ready: oneshot::Sender<()>,
) -> Result<(), String> {
    let mut ready = Some(ready);
    let endpoint = Endpoint::new(&state).ok();
    let mut hub = None;
    let mut probe = probe::Probe::new(state);
    // The baseline probe below already covers startup. An immediate interval tick
    // could obscure a racing publication with a redundant Resync.
    let mut reconcile =
        tokio::time::interval_at(tokio::time::Instant::now() + RECONCILE, RECONCILE);
    reconcile.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
    // Establish the cheap persistent baseline before the caller reads its snapshot.
    let _ = probe.changed().await;
    loop {
        if cancel.is_cancelled() {
            break;
        }
        if let Some(endpoint) = &endpoint {
            if hub
                .as_ref()
                .is_some_and(|task: &tokio::task::JoinHandle<()>| task.is_finished())
            {
                hub = None;
            }
            if hub.is_none()
                && let Ok(Some(owner)) = endpoint.elect()
                && let Ok(listener) = endpoint.listen()
            {
                let endpoint = endpoint.clone();
                let token = cancel.child_token();
                hub = Some(tokio::spawn(async move {
                    let _ = serve(endpoint, listener, owner, token).await;
                }));
            }
            let connection = tokio::time::timeout(DELIVERY_TIMEOUT, connect(endpoint, true));
            tokio::select! {
                _ = cancel.cancelled() => break,
                result = connection => {
                    if let Ok(Ok((stream, epoch))) = result {
                        inbox.add(Change::Resync);
                        #[cfg(test)]
                        inbox.registration.send_replace(Some(epoch.clone()));
                        if let Some(ready) = ready.take() {
                            let _ = ready.send(());
                        }
                        let receive = receive(stream, epoch, inbox.clone());
                        tokio::pin!(receive);
                        loop {
                            tokio::select! {
                                _ = cancel.cancelled() => break,
                                _ = &mut receive => {
                                    #[cfg(test)]
                                    inbox.registration.send_replace(None);
                                    inbox.add(Change::Resync);
                                    break;
                                }
                                // A commit no hint reported is read like one; only a
                                // replaced database is read afresh.
                                _ = reconcile.tick() => match probe.changed().await {
                                    probe::Probed::Same => {}
                                    probe::Probed::Written => inbox.add(Change::StateInvalidated),
                                    probe::Probed::Replaced => inbox.add(Change::Resync),
                                }
                            }
                        }
                    }
                }
            }
        }
        if let Some(ready) = ready.take() {
            inbox.add(Change::Resync);
            let _ = ready.send(());
        }
        tokio::select! {
            _ = cancel.cancelled() => break,
            _ = reconcile.tick() => {
                let _ = probe.changed().await;
                // No secure transport: low-frequency authoritative reconciliation also
                // recovers session-only writes, which SQLite data_version cannot observe.
                inbox.add(Change::Resync);
            },
            _ = tokio::time::sleep(RECONNECT) => {},
        }
    }
    if let Some(task) = hub {
        // Its child token is cancelled too; let serve join every client before releasing IPC.
        let _ = crate::task::joined(task.await);
    }
    probe.close().await
}
async fn receive(
    mut stream: transport::Stream,
    epoch: crate::config::HubEpoch,
    inbox: Arc<Inbox>,
) -> std::io::Result<()> {
    let mut previous = None;
    loop {
        match read_frame(&mut stream).await? {
            Frame::Hint {
                epoch: incoming,
                sequence,
                change,
            } if incoming == epoch => {
                if previous.is_some_and(|previous: u64| sequence != previous.wrapping_add(1)) {
                    inbox.add(Change::Resync);
                }
                previous = Some(sequence);
                inbox.add(change);
                #[cfg(test)]
                inbox.received.send_replace(Some(sequence));
            }
            _ => return Err(std::io::ErrorKind::InvalidData.into()),
        }
    }
}

#[derive(Clone)]
struct Hint {
    sequence: u64,
    change: Change,
}
async fn serve(
    endpoint: Endpoint,
    mut listener: transport::Listener,
    _owner: std::fs::File,
    cancel: CancellationToken,
) -> std::io::Result<()> {
    let epoch: crate::config::HubEpoch = crate::agent::uuid()
        .map_err(std::io::Error::other)?
        .parse()
        .expect("UUID hub epoch");
    let (sender, _) = broadcast::channel(MAX_BACKLOG);
    let sequence = Arc::new(Mutex::new(0_u64));
    let mut clients = JoinSet::new();
    loop {
        tokio::select! {
            _ = cancel.cancelled() => break,
            Some(result) = clients.join_next(), if !clients.is_empty() => {
                let _ = crate::task::joined(result);
            },
            result = listener.accept(), if clients.len() < MAX_CLIENTS => {
                let stream = result?;
                let (identity, epoch, sender, sequence) = (
                    endpoint.identity.clone(),
                    epoch.clone(),
                    sender.clone(),
                    sequence.clone(),
                );
                clients.spawn(async move {
                    let _ = client(stream, identity, epoch, sender, sequence).await;
                });
            }
        }
    }
    clients.abort_all();
    while let Some(result) = clients.join_next().await {
        let _ = crate::task::joined(result);
    }
    Ok(())
}
async fn client(
    mut stream: transport::Stream,
    identity: crate::config::EndpointId,
    epoch: crate::config::HubEpoch,
    sender: broadcast::Sender<Hint>,
    sequence: Arc<Mutex<u64>>,
) -> std::io::Result<()> {
    // Peer lookup can fail after a short-lived connection closes (Darwin ENOTCONN).
    // Reject only that client, never tear down the listener or other subscribers.
    transport::validate_peer(&stream)?;
    let hello = tokio::time::timeout(DELIVERY_TIMEOUT, read_frame(&mut stream)).await??;
    let Frame::Hello {
        version: VERSION,
        identity: incoming,
        subscriber,
    } = hello
    else {
        return Err(std::io::ErrorKind::InvalidData.into());
    };
    if incoming != identity {
        return Err(std::io::ErrorKind::PermissionDenied.into());
    }
    // Register before acknowledging. Broadcast retains any hint that races the baseline read.
    let mut receiver = sender.subscribe();
    tokio::time::timeout(
        DELIVERY_TIMEOUT,
        write_frame(
            &mut stream,
            &Frame::Registered {
                epoch: epoch.clone(),
            },
        ),
    )
    .await??;
    if subscriber {
        let (mut reader, mut writer) = tokio::io::split(stream);
        let outgoing = async {
            loop {
                let hint = match receiver.recv().await {
                    Ok(hint) => hint,
                    Err(broadcast::error::RecvError::Lagged(_)) => Hint {
                        sequence: *sequence.lock().unwrap(),
                        change: Change::Resync,
                    },
                    Err(_) => return Ok::<_, std::io::Error>(()),
                };
                // A slow client loses only hints, not memory or the hub's other clients.
                tokio::time::timeout(
                    DELIVERY_TIMEOUT,
                    write_frame(
                        &mut writer,
                        &Frame::Hint {
                            epoch: epoch.clone(),
                            sequence: hint.sequence,
                            change: hint.change,
                        },
                    ),
                )
                .await??;
            }
        };
        tokio::select! { result = outgoing => result, _ = reader.read_u8() => Ok(()) }
    } else {
        loop {
            let frame = tokio::time::timeout(DELIVERY_TIMEOUT, read_frame(&mut stream)).await??;
            let Frame::Publish(change) = frame else {
                return Err(std::io::ErrorKind::InvalidData.into());
            };
            {
                // Serialize sequence assignment and broadcast, including concurrent writers.
                let mut sequence = sequence.lock().unwrap();
                *sequence = sequence.wrapping_add(1);
                let _ = sender.send(Hint {
                    sequence: *sequence,
                    change,
                });
            }
            tokio::time::timeout(
                DELIVERY_TIMEOUT,
                write_frame(&mut stream, &Frame::Delivered),
            )
            .await??;
        }
    }
}