1use std::sync::{Arc, Weak};
4
5use khive_db::pool::{TimeoutSinkCloseError, TimeoutSinkCloseHandle};
6use khive_db::{ConnectionPool, StorageBackend};
7
8use super::KhiveRuntime;
9use crate::audit_batch::AuditTerminalReason;
10use crate::VerbRegistry;
11
12#[derive(Clone, Copy, Debug, Default)]
17pub struct RuntimeCloseHostTasks {
18 pub daemon_checkpoint: bool,
19 pub mcp_session_sweep: bool,
20}
21
22#[derive(Clone, Copy, Debug, PartialEq, Eq)]
24pub enum RuntimeClosePhase {
25 Ready,
26 Closing,
27 Closed,
28}
29
30#[derive(Clone, Copy, Debug, PartialEq, Eq)]
32pub enum RuntimeCloseRootKind {
33 Pool,
34 Backend,
35 AuditStore,
36 TimeoutSinkPool,
37 TimeoutSinkWriter,
38}
39
40#[derive(Clone, Debug, PartialEq, Eq)]
43pub struct RuntimeCloseRoot {
44 pub kind: RuntimeCloseRootKind,
45 pub strong_count: usize,
46}
47
48#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
50pub enum RuntimeCloseError {
51 #[error("runtime close cannot settle owner: {0}")]
52 Refused(&'static str),
53 #[error("committed runtime close is incomplete; surviving roots: {roots:?}")]
54 Incomplete { roots: Vec<RuntimeCloseRoot> },
55 #[error("runtime close cannot prove writer completion: its join was taken by another owner")]
56 WriterJoinUnavailable,
57 #[error("runtime writer did not complete normally: {0}")]
58 WriterFailed(String),
59 #[error("runtime timeout sink did not complete normally")]
60 SinkFailed,
61 #[error("audit drained physically but reported terminal failure: {0:?}")]
62 AuditFailed(AuditTerminalReason),
63}
64
65struct WriterClose {
66 pool: Weak<ConnectionPool>,
67 join: Option<tokio::task::JoinHandle<()>>,
68 witness: Option<tokio::task::AbortHandle>,
69 result: Option<Result<(), RuntimeCloseError>>,
70 sink: Option<TimeoutSinkCloseHandle>,
71}
72
73pub struct RuntimeCloseOwner {
91 registry: Option<VerbRegistry>,
92 phase: RuntimeClosePhase,
93 prepared_pools: Vec<Arc<ConnectionPool>>,
94 backend_roots: Vec<Weak<StorageBackend>>,
95 audit_store_root: Option<Weak<dyn khive_storage::EventStore>>,
96 runtimes: Vec<KhiveRuntime>,
97 host_tasks: RuntimeCloseHostTasks,
98 writers: Option<Vec<WriterClose>>,
99 audit_error: Option<AuditTerminalReason>,
100 completed: Option<Result<(), RuntimeCloseError>>,
101}
102
103impl RuntimeCloseOwner {
104 pub fn new(
105 registry: VerbRegistry,
106 runtimes: Vec<KhiveRuntime>,
107 host_tasks: RuntimeCloseHostTasks,
108 ) -> Self {
109 Self {
110 registry: Some(registry),
111 phase: RuntimeClosePhase::Ready,
112 prepared_pools: Vec::new(),
113 backend_roots: Vec::new(),
114 audit_store_root: None,
115 runtimes,
116 host_tasks,
117 writers: None,
118 audit_error: None,
119 completed: None,
120 }
121 }
122
123 pub fn phase(&self) -> RuntimeClosePhase {
124 self.phase
125 }
126
127 pub fn open_registry(&self) -> Option<&VerbRegistry> {
130 if self.phase == RuntimeClosePhase::Ready {
131 self.registry.as_ref()
132 } else {
133 None
134 }
135 }
136
137 pub fn open_runtimes(&self) -> Option<&[KhiveRuntime]> {
139 (self.phase == RuntimeClosePhase::Ready).then_some(self.runtimes.as_slice())
140 }
141
142 pub fn is_closed(&self) -> bool {
144 self.completed.as_ref().is_some_and(Result::is_ok)
145 }
146
147 pub fn writer_tasks_finished(&self) -> bool {
149 self.writers.as_ref().is_some_and(|writers| {
150 writers.iter().all(|writer| {
151 writer.witness.as_ref().map_or_else(
152 || writer.result.as_ref().is_some_and(Result::is_ok),
153 tokio::task::AbortHandle::is_finished,
154 )
155 })
156 })
157 }
158
159 fn pools_and_preflight(&self) -> Result<Vec<Arc<ConnectionPool>>, RuntimeCloseError> {
160 use RuntimeCloseError::Refused;
161 if self.host_tasks.daemon_checkpoint {
162 return Err(Refused("daemon checkpoint loop"));
163 }
164 if self.host_tasks.mcp_session_sweep {
165 return Err(Refused("MCP session sweep"));
166 }
167 if self.runtimes.is_empty() {
168 return Err(Refused("empty runtime bundle"));
169 }
170 let mut pools = Vec::new();
171 for runtime in &self.runtimes {
172 if runtime.config.events_split.is_some() {
173 return Err(Refused("split/socket events registry"));
174 }
175 let registry = runtime
178 .embedder_registry
179 .read()
180 .map_err(|_| Refused("poisoned embedder registry"))?;
181 if !registry.names().is_empty() || runtime.core_embedders.is_some() {
182 return Err(Refused("embedder initialization and worker thread"));
183 }
184 drop(registry);
185 if runtime
186 .note_search_ann_provider
187 .read()
188 .map_err(|_| Refused("poisoned ANN provider registry"))?
189 .is_some()
190 || runtime
191 .note_mutation_hook
192 .read()
193 .map_err(|_| Refused("poisoned note mutation hook"))?
194 .is_some()
195 {
196 return Err(Refused("pack ANN tasks and mapped indexes"));
197 }
198 if runtime.blob_hydrator.get().is_some() {
199 return Err(Refused("blob hydration supervisors and blocking work"));
200 }
201 if runtime.core_backend.is_some() {
202 return Err(Refused("secondary backend runtime"));
203 }
204 let main = runtime.backend.pool_arc();
205 for opened in runtime.diagnostic_backends().iter() {
206 if !Arc::ptr_eq(&main, &opened.pool) {
207 return Err(Refused("additional diagnostic backend"));
208 }
209 }
210 if main.checkpoint_owner_present() {
211 return Err(Refused("claimed daemon checkpoint owner"));
212 }
213 if !pools.iter().any(|pool| Arc::ptr_eq(pool, &main)) {
214 pools.push(main);
215 }
216 }
217 if pools.len() != 1 {
218 return Err(Refused("multiple backend pools"));
219 }
220 Ok(pools)
221 }
222
223 pub async fn close(&mut self) -> Result<(), RuntimeCloseError> {
231 if let Some(result) = &self.completed {
232 return result.clone();
233 }
234 if self.writers.is_none() {
235 if self.phase == RuntimeClosePhase::Ready {
238 self.prepared_pools = self.pools_and_preflight()?;
239 for runtime in &self.runtimes {
240 let weak = Arc::downgrade(&runtime.backend);
241 if !self
242 .backend_roots
243 .iter()
244 .any(|root| Weak::ptr_eq(root, &weak))
245 {
246 self.backend_roots.push(weak);
247 }
248 }
249 self.audit_store_root = self
250 .registry
251 .as_ref()
252 .and_then(VerbRegistry::event_store)
253 .map(|store| Arc::downgrade(&store));
254 self.phase = RuntimeClosePhase::Closing;
255 }
256 if let Some(batch) = self
257 .registry
258 .as_ref()
259 .and_then(VerbRegistry::audit_batch_handle)
260 {
261 self.audit_error = batch.close_and_wait_for_physical_completion().await.err();
262 }
263 let pools = std::mem::take(&mut self.prepared_pools);
264 let writers = pools
265 .iter()
266 .map(|pool| {
267 let join = pool.take_writer_task_join();
268 let result = if join.is_some() {
269 None
270 } else if pool.writer_task_join_was_stored() {
271 Some(Err(RuntimeCloseError::WriterJoinUnavailable))
272 } else {
273 Some(Ok(()))
274 };
275 WriterClose {
276 pool: Arc::downgrade(pool),
277 witness: join.as_ref().map(tokio::task::JoinHandle::abort_handle),
278 join,
279 result,
280 sink: pool.timeout_sink_close_handle(),
281 }
282 })
283 .collect();
284 self.writers = Some(writers);
287 self.registry = None;
288 self.runtimes.clear();
289 drop(pools);
290 }
291 let writers = self.writers.as_mut().expect("close state initialized");
292 let mut roots = Vec::new();
293 for writer in writers.iter() {
294 add_surviving_root(
295 &mut roots,
296 RuntimeCloseRootKind::Pool,
297 writer.pool.strong_count(),
298 );
299 }
300 for backend in &self.backend_roots {
301 add_surviving_root(
302 &mut roots,
303 RuntimeCloseRootKind::Backend,
304 backend.strong_count(),
305 );
306 }
307 if let Some(store) = &self.audit_store_root {
308 add_surviving_root(
309 &mut roots,
310 RuntimeCloseRootKind::AuditStore,
311 store.strong_count(),
312 );
313 }
314 if !roots.is_empty() {
315 return Err(RuntimeCloseError::Incomplete { roots });
316 }
317 for writer in writers.iter_mut() {
318 if let Some(join) = writer.join.as_mut() {
319 let result = join
320 .await
321 .map_err(|error| RuntimeCloseError::WriterFailed(error.to_string()));
322 writer.result = Some(result);
323 writer.join = None;
324 }
325 }
326 for writer in writers.iter() {
330 writer
331 .result
332 .as_ref()
333 .expect("writer outcome recorded")
334 .clone()?;
335 }
336 for writer in writers.iter_mut() {
337 if let Some(sink) = writer.sink.as_mut() {
338 sink.close().await.map_err(|error| match error {
339 TimeoutSinkCloseError::Busy { pools, writers } => {
340 let mut roots = Vec::new();
341 add_surviving_root(
342 &mut roots,
343 RuntimeCloseRootKind::TimeoutSinkPool,
344 pools,
345 );
346 add_surviving_root(
347 &mut roots,
348 RuntimeCloseRootKind::TimeoutSinkWriter,
349 writers,
350 );
351 RuntimeCloseError::Incomplete { roots }
352 }
353 _ => RuntimeCloseError::SinkFailed,
354 })?;
355 }
356 }
357 let result = self
358 .audit_error
359 .map_or(Ok(()), |error| Err(RuntimeCloseError::AuditFailed(error)));
360 if result.is_ok() {
361 self.phase = RuntimeClosePhase::Closed;
362 }
363 self.completed = Some(result.clone());
364 result
365 }
366}
367
368fn add_surviving_root(
369 roots: &mut Vec<RuntimeCloseRoot>,
370 kind: RuntimeCloseRootKind,
371 strong_count: usize,
372) {
373 if strong_count != 0 {
374 roots.push(RuntimeCloseRoot { kind, strong_count });
375 }
376}
377
378#[cfg(test)]
379#[path = "close_tests.rs"]
380mod tests;