Skip to main content

khive_runtime/runtime/
close.rs

1//! Host-owned close state for a quiescent, single-backend runtime slot.
2
3use std::sync::{Arc, Weak};
4
5use khive_db::pool::{TimeoutSinkCloseError, TimeoutSinkCloseHandle};
6use khive_db::{ConnectionPool, StorageBackend};
7
8use super::KhiveRuntime;
9use crate::audit_batch::AuditTerminalReason;
10use crate::VerbRegistry;
11
12/// Host resources that are outside this runtime's join ownership.
13///
14/// Hosts must stop request admission before constructing a close owner and
15/// accurately declare these tasks. This API never cancels process-wide tasks.
16#[derive(Clone, Copy, Debug, Default)]
17pub struct RuntimeCloseHostTasks {
18    pub daemon_checkpoint: bool,
19    pub mcp_session_sweep: bool,
20}
21
22/// The reversible preflight and irreversible, resumable teardown boundary.
23#[derive(Clone, Copy, Debug, PartialEq, Eq)]
24pub enum RuntimeClosePhase {
25    Ready,
26    Closing,
27    Closed,
28}
29
30/// A root for which the close owner retained a weak completion witness.
31#[derive(Clone, Copy, Debug, PartialEq, Eq)]
32pub enum RuntimeCloseRootKind {
33    Pool,
34    Backend,
35    AuditStore,
36    TimeoutSinkPool,
37    TimeoutSinkWriter,
38}
39
40/// Strong references still keeping an observed root alive after teardown commits.
41/// Counts are snapshots, not an inventory of the identities holding references.
42#[derive(Clone, Debug, PartialEq, Eq)]
43pub struct RuntimeCloseRoot {
44    pub kind: RuntimeCloseRootKind,
45    pub strong_count: usize,
46}
47
48/// A close refusal or incomplete release. None of these means eviction is safe.
49#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
50pub enum RuntimeCloseError {
51    #[error("runtime close cannot settle owner: {0}")]
52    Refused(&'static str),
53    #[error("committed runtime close is incomplete; surviving roots: {roots:?}")]
54    Incomplete { roots: Vec<RuntimeCloseRoot> },
55    #[error("runtime close cannot prove writer completion: its join was taken by another owner")]
56    WriterJoinUnavailable,
57    #[error("runtime writer did not complete normally: {0}")]
58    WriterFailed(String),
59    #[error("runtime timeout sink did not complete normally")]
60    SinkFailed,
61    #[error("audit drained physically but reported terminal failure: {0:?}")]
62    AuditFailed(AuditTerminalReason),
63}
64
65struct WriterClose {
66    pool: Weak<ConnectionPool>,
67    join: Option<tokio::task::JoinHandle<()>>,
68    witness: Option<tokio::task::AbortHandle>,
69    result: Option<Result<(), RuntimeCloseError>>,
70    sink: Option<TimeoutSinkCloseHandle>,
71}
72
73/// Own every host-supplied runtime clone and its matching registry during eviction.
74///
75/// The host must first stop admitting requests and wait for its active request
76/// counter. Transfer the registry built from these runtimes and *all* host-held
77/// clones. A forgotten pool/capability returns `Incomplete` after teardown has
78/// committed; drop that holder and retry. Do not reopen the root until `Ok`.
79/// Configuration `Refused` results preserve the fully usable Ready state. The bundle retains writer joins across cancelled `close` futures.
80/// Dropping the bundle itself is still best effort, not successful close.
81///
82/// The supported scope is a single backend without embeddings, ANN, blob
83/// hydration, split events, or host background tasks. Those owners currently
84/// lack a tenant-scoped physical completion barrier and are explicitly refused.
85/// A successful return covers the owned audit tasks, SQLite pool/writer and
86/// last-store timeout sink; it is not a census of arbitrary host/provider files.
87///
88/// Registry packs and its audit store must belong to the supplied runtime slot.
89/// Custom packs must not hide independent file or task owners in this bundle.
90pub struct RuntimeCloseOwner {
91    registry: Option<VerbRegistry>,
92    phase: RuntimeClosePhase,
93    prepared_pools: Vec<Arc<ConnectionPool>>,
94    backend_roots: Vec<Weak<StorageBackend>>,
95    audit_store_root: Option<Weak<dyn khive_storage::EventStore>>,
96    runtimes: Vec<KhiveRuntime>,
97    host_tasks: RuntimeCloseHostTasks,
98    writers: Option<Vec<WriterClose>>,
99    audit_error: Option<AuditTerminalReason>,
100    completed: Option<Result<(), RuntimeCloseError>>,
101}
102
103impl RuntimeCloseOwner {
104    pub fn new(
105        registry: VerbRegistry,
106        runtimes: Vec<KhiveRuntime>,
107        host_tasks: RuntimeCloseHostTasks,
108    ) -> Self {
109        Self {
110            registry: Some(registry),
111            phase: RuntimeClosePhase::Ready,
112            prepared_pools: Vec::new(),
113            backend_roots: Vec::new(),
114            audit_store_root: None,
115            runtimes,
116            host_tasks,
117            writers: None,
118            audit_error: None,
119            completed: None,
120        }
121    }
122
123    pub fn phase(&self) -> RuntimeClosePhase {
124        self.phase
125    }
126
127    /// The registry remains usable after reversible refusal. It is unavailable
128    /// after teardown commits, including while a close future is cancelled.
129    pub fn open_registry(&self) -> Option<&VerbRegistry> {
130        if self.phase == RuntimeClosePhase::Ready {
131            self.registry.as_ref()
132        } else {
133            None
134        }
135    }
136
137    /// Transferred handles remain usable after reversible refusal only.
138    pub fn open_runtimes(&self) -> Option<&[KhiveRuntime]> {
139        (self.phase == RuntimeClosePhase::Ready).then_some(self.runtimes.as_slice())
140    }
141
142    /// Whether close has successfully completed; errors never mark it closed.
143    pub fn is_closed(&self) -> bool {
144        self.completed.as_ref().is_some_and(Result::is_ok)
145    }
146
147    /// Observe actual writer task completion without aborting or taking a join.
148    pub fn writer_tasks_finished(&self) -> bool {
149        self.writers.as_ref().is_some_and(|writers| {
150            writers.iter().all(|writer| {
151                writer.witness.as_ref().map_or_else(
152                    || writer.result.as_ref().is_some_and(Result::is_ok),
153                    tokio::task::AbortHandle::is_finished,
154                )
155            })
156        })
157    }
158
159    fn pools_and_preflight(&self) -> Result<Vec<Arc<ConnectionPool>>, RuntimeCloseError> {
160        use RuntimeCloseError::Refused;
161        if self.host_tasks.daemon_checkpoint {
162            return Err(Refused("daemon checkpoint loop"));
163        }
164        if self.host_tasks.mcp_session_sweep {
165            return Err(Refused("MCP session sweep"));
166        }
167        if self.runtimes.is_empty() {
168            return Err(Refused("empty runtime bundle"));
169        }
170        let mut pools = Vec::new();
171        for runtime in &self.runtimes {
172            if runtime.config.events_split.is_some() {
173                return Err(Refused("split/socket events registry"));
174            }
175            // Read the actual registry, not the request-filtered public model
176            // list: request embedder exclusions must not hide a live worker.
177            let registry = runtime
178                .embedder_registry
179                .read()
180                .map_err(|_| Refused("poisoned embedder registry"))?;
181            if !registry.names().is_empty() || runtime.core_embedders.is_some() {
182                return Err(Refused("embedder initialization and worker thread"));
183            }
184            drop(registry);
185            if runtime
186                .note_search_ann_provider
187                .read()
188                .map_err(|_| Refused("poisoned ANN provider registry"))?
189                .is_some()
190                || runtime
191                    .note_mutation_hook
192                    .read()
193                    .map_err(|_| Refused("poisoned note mutation hook"))?
194                    .is_some()
195            {
196                return Err(Refused("pack ANN tasks and mapped indexes"));
197            }
198            if runtime.blob_hydrator.get().is_some() {
199                return Err(Refused("blob hydration supervisors and blocking work"));
200            }
201            if runtime.core_backend.is_some() {
202                return Err(Refused("secondary backend runtime"));
203            }
204            let main = runtime.backend.pool_arc();
205            for opened in runtime.diagnostic_backends().iter() {
206                if !Arc::ptr_eq(&main, &opened.pool) {
207                    return Err(Refused("additional diagnostic backend"));
208                }
209            }
210            if main.checkpoint_owner_present() {
211                return Err(Refused("claimed daemon checkpoint owner"));
212            }
213            if !pools.iter().any(|pool| Arc::ptr_eq(pool, &main)) {
214                pools.push(main);
215            }
216        }
217        if pools.len() != 1 {
218            return Err(Refused("multiple backend pools"));
219        }
220        Ok(pools)
221    }
222
223    /// Stop audit admission, await physical append/reaper completion, drop all
224    /// transferred roots, join the writer, then close the final store's sink.
225    ///
226    /// Cancellation only interrupts this wait. Retain this owner and call again
227    /// to resume; no accepted append or native write is aborted. A second call
228    /// after success returns `Ok(())`. A permanently stuck native operation can
229    /// keep close pending; a caller timeout is not evidence of settlement.
230    pub async fn close(&mut self) -> Result<(), RuntimeCloseError> {
231        if let Some(result) = &self.completed {
232            return result.clone();
233        }
234        if self.writers.is_none() {
235            // Refuse unsupported configurations before stopping audit admission
236            // or consuming any roots/joins, so the host retains its owners.
237            if self.phase == RuntimeClosePhase::Ready {
238                self.prepared_pools = self.pools_and_preflight()?;
239                for runtime in &self.runtimes {
240                    let weak = Arc::downgrade(&runtime.backend);
241                    if !self
242                        .backend_roots
243                        .iter()
244                        .any(|root| Weak::ptr_eq(root, &weak))
245                    {
246                        self.backend_roots.push(weak);
247                    }
248                }
249                self.audit_store_root = self
250                    .registry
251                    .as_ref()
252                    .and_then(VerbRegistry::event_store)
253                    .map(|store| Arc::downgrade(&store));
254                self.phase = RuntimeClosePhase::Closing;
255            }
256            if let Some(batch) = self
257                .registry
258                .as_ref()
259                .and_then(VerbRegistry::audit_batch_handle)
260            {
261                self.audit_error = batch.close_and_wait_for_physical_completion().await.err();
262            }
263            let pools = std::mem::take(&mut self.prepared_pools);
264            let writers = pools
265                .iter()
266                .map(|pool| {
267                    let join = pool.take_writer_task_join();
268                    let result = if join.is_some() {
269                        None
270                    } else if pool.writer_task_join_was_stored() {
271                        Some(Err(RuntimeCloseError::WriterJoinUnavailable))
272                    } else {
273                        Some(Ok(()))
274                    };
275                    WriterClose {
276                        pool: Arc::downgrade(pool),
277                        witness: join.as_ref().map(tokio::task::JoinHandle::abort_handle),
278                        join,
279                        result,
280                        sink: pool.timeout_sink_close_handle(),
281                    }
282                })
283                .collect();
284            // Publish all wait state before releasing the roots. No await may
285            // intervene between taking a join and retaining it on this owner.
286            self.writers = Some(writers);
287            self.registry = None;
288            self.runtimes.clear();
289            drop(pools);
290        }
291        let writers = self.writers.as_mut().expect("close state initialized");
292        let mut roots = Vec::new();
293        for writer in writers.iter() {
294            add_surviving_root(
295                &mut roots,
296                RuntimeCloseRootKind::Pool,
297                writer.pool.strong_count(),
298            );
299        }
300        for backend in &self.backend_roots {
301            add_surviving_root(
302                &mut roots,
303                RuntimeCloseRootKind::Backend,
304                backend.strong_count(),
305            );
306        }
307        if let Some(store) = &self.audit_store_root {
308            add_surviving_root(
309                &mut roots,
310                RuntimeCloseRootKind::AuditStore,
311                store.strong_count(),
312            );
313        }
314        if !roots.is_empty() {
315            return Err(RuntimeCloseError::Incomplete { roots });
316        }
317        for writer in writers.iter_mut() {
318            if let Some(join) = writer.join.as_mut() {
319                let result = join
320                    .await
321                    .map_err(|error| RuntimeCloseError::WriterFailed(error.to_string()));
322                writer.result = Some(result);
323                writer.join = None;
324            }
325        }
326        // A missing/failed writer join is never promoted to physical success.
327        // Preserve the evidence on repeated calls rather than mistaking None
328        // in the already-consumed pool slot for a completed writer.
329        for writer in writers.iter() {
330            writer
331                .result
332                .as_ref()
333                .expect("writer outcome recorded")
334                .clone()?;
335        }
336        for writer in writers.iter_mut() {
337            if let Some(sink) = writer.sink.as_mut() {
338                sink.close().await.map_err(|error| match error {
339                    TimeoutSinkCloseError::Busy { pools, writers } => {
340                        let mut roots = Vec::new();
341                        add_surviving_root(
342                            &mut roots,
343                            RuntimeCloseRootKind::TimeoutSinkPool,
344                            pools,
345                        );
346                        add_surviving_root(
347                            &mut roots,
348                            RuntimeCloseRootKind::TimeoutSinkWriter,
349                            writers,
350                        );
351                        RuntimeCloseError::Incomplete { roots }
352                    }
353                    _ => RuntimeCloseError::SinkFailed,
354                })?;
355            }
356        }
357        let result = self
358            .audit_error
359            .map_or(Ok(()), |error| Err(RuntimeCloseError::AuditFailed(error)));
360        if result.is_ok() {
361            self.phase = RuntimeClosePhase::Closed;
362        }
363        self.completed = Some(result.clone());
364        result
365    }
366}
367
368fn add_surviving_root(
369    roots: &mut Vec<RuntimeCloseRoot>,
370    kind: RuntimeCloseRootKind,
371    strong_count: usize,
372) {
373    if strong_count != 0 {
374        roots.push(RuntimeCloseRoot { kind, strong_count });
375    }
376}
377
378#[cfg(test)]
379#[path = "close_tests.rs"]
380mod tests;