1use std::any::Any;
4
5use khive_db::pending_index::{
6 self, PendingEntityIndex, PendingIndexTarget, PendingIndexTargetKind,
7};
8use khive_storage::{AtomicUnitOp, Entity, SqlStatement, StorageError, WriterTaskRequestState};
9use serde::Serialize;
10
11use crate::retrieval::EmbeddingTruncationReport;
12use crate::{KhiveRuntime, NamespaceToken, PostCommitDegradation, RuntimeError, RuntimeResult};
13
14#[cfg(test)]
15tokio::task_local! {
16 static UNKNOWN_AFTER_PUBLISH: bool;
17}
18
19#[derive(Clone, Debug, Serialize)]
21pub struct EntityIndexFailure {
22 pub target: PendingIndexTargetKind,
23 pub stage: String,
25}
26
27#[derive(Clone, Debug, Serialize)]
29pub struct EntityIndexReport {
30 pub status: &'static str,
31 pub unfinished: Vec<EntityIndexFailure>,
32}
33
34impl EntityIndexReport {
35 pub(super) fn complete(required: bool) -> Self {
36 Self {
37 status: if required {
38 "complete"
39 } else {
40 "not_requested"
41 },
42 unfinished: Vec::new(),
43 }
44 }
45
46 fn unfinished(&mut self, target: &PendingIndexTargetKind, stage: &str) {
47 self.status = "unfinished";
48 self.unfinished.push(EntityIndexFailure {
49 target: target.clone(),
50 stage: stage.into(),
51 });
52 }
53}
54
55#[derive(Clone, Debug, Serialize)]
57pub struct EntityUpdateReport {
58 pub embedding: EmbeddingTruncationReport,
59 pub indexing: EntityIndexReport,
60 pub post_commit_degradations: Vec<PostCommitDegradation>,
61}
62
63impl EntityUpdateReport {
64 pub(super) fn complete(embedding: EmbeddingTruncationReport, required: bool) -> Self {
65 Self {
66 embedding,
67 indexing: EntityIndexReport::complete(required),
68 post_commit_degradations: Vec::new(),
69 }
70 }
71
72 pub(super) fn into_legacy(
74 self,
75 entity: Entity,
76 ) -> RuntimeResult<(Entity, EmbeddingTruncationReport)> {
77 let mut degradations = self.post_commit_degradations;
78 degradations.extend(self.indexing.unfinished.into_iter().map(|failure| {
79 PostCommitDegradation {
80 stage: "entity_index_unfinished",
81 error: serde_json::json!(failure).to_string(),
82 }
83 }));
84 crate::operations::legacy_post_commit_result(
85 "update_entity",
86 entity.id,
87 (entity, self.embedding),
88 degradations,
89 )
90 }
91}
92
93fn unsettled(error: &RuntimeError) -> bool {
94 error
95 .writer_task_failure_context()
96 .is_some_and(|context| context.request_state == WriterTaskRequestState::SideEffectsUnknown)
97 || matches!(error.refusal_source(), RuntimeError::Storage(error)
98 if error.sqlite_write_failure().is_some_and(|failure| failure.settlement_unknown))
99}
100
101impl KhiveRuntime {
102 pub(super) async fn publish_pending_entity_target(
105 &self,
106 observed: &PendingEntityIndex,
107 target: &PendingIndexTargetKind,
108 statements: Vec<SqlStatement>,
109 ) -> RuntimeResult<bool> {
110 let observed = observed.clone();
111 let target = target.clone();
112 let op: AtomicUnitOp = Box::new(move |writer| {
113 Box::pin(async move {
114 let Some(current) = pending_index::read(writer, observed.entity_id).await? else {
115 return Ok(Box::new(false) as Box<dyn Any + Send>);
116 };
117 if current.generation != observed.generation
118 || current.namespace != observed.namespace
119 || current.source_version != observed.source_version
120 || !current.targets.iter().any(|entry| {
121 entry.kind == target && !entry.done && !entry.kind.is_blocked()
122 })
123 || !pending_index::live_matches(writer, ¤t).await?
124 {
125 return Ok(Box::new(false) as Box<dyn Any + Send>);
126 }
127 for statement in statements {
128 writer.execute(statement).await?;
129 }
130 if !pending_index::finish(writer, current.entity_id, current.generation, &target)
131 .await?
132 {
133 return Err(StorageError::Internal(
135 "pending index consumption guard failed".into(),
136 ));
137 }
138 Ok(Box::new(true) as Box<dyn Any + Send>)
139 })
140 });
141 let outcome = self.sql().atomic_unit(op).await?;
142 #[cfg(test)]
143 if UNKNOWN_AFTER_PUBLISH
144 .try_with(|enabled| *enabled)
145 .unwrap_or(false)
146 {
147 return Err(RuntimeError::Storage(StorageError::writer_task_terminated(
148 WriterTaskRequestState::SideEffectsUnknown,
149 )));
150 }
151 outcome
152 .downcast::<bool>()
153 .map(|result| *result)
154 .map_err(|_| RuntimeError::Internal("invalid pending index publication outcome".into()))
155 }
156
157 async fn record_pending_entity_failure(
158 &self,
159 observed: &PendingEntityIndex,
160 target: &PendingIndexTarget,
161 category: &'static str,
162 ) -> RuntimeResult<bool> {
163 let id = observed.entity_id;
164 let generation = observed.generation;
165 let kind = target.kind.clone();
166 let next_eligible_at = target.next_eligible_at;
169 let op: AtomicUnitOp = Box::new(move |writer| {
170 Box::pin(async move {
171 let recorded = pending_index::record_failure(
172 writer,
173 id,
174 generation,
175 &kind,
176 category,
177 next_eligible_at,
178 )
179 .await?;
180 Ok(Box::new(recorded) as Box<dyn Any + Send>)
181 })
182 });
183 self.sql()
184 .atomic_unit(op)
185 .await?
186 .downcast::<bool>()
187 .map(|result| *result)
188 .map_err(|_| RuntimeError::Internal("invalid pending index failure outcome".into()))
189 }
190
191 async fn prepare_pending_entity_target(
192 &self,
193 token: &NamespaceToken,
194 entity: &Entity,
195 target: &PendingIndexTargetKind,
196 snapshot: &[(String, usize)],
197 report: &mut EmbeddingTruncationReport,
198 ) -> Result<Vec<SqlStatement>, (&'static str, RuntimeError)> {
199 match target {
200 PendingIndexTargetKind::Fts => {
201 #[cfg(test)]
202 if crate::operations::consume_fts_fail_fault(&entity.namespace) {
203 return Err((
204 "fts_prepare",
205 RuntimeError::Internal("injected FTS failure".into()),
206 ));
207 }
208 self.text(token).map_err(|error| ("fts_prepare", error))?;
209 let doc = super::entity_fts_document(entity);
210 Ok(khive_db::stores::text::delete_document_statements(
211 "fts_entities",
212 &entity.namespace,
213 entity.id,
214 )
215 .into_iter()
216 .chain(khive_db::stores::text::insert_document_statements(
217 "fts_entities",
218 &doc,
219 ))
220 .collect())
221 }
222 PendingIndexTargetKind::Vector { model, dimensions } => {
223 let Some((_, captured_dimensions)) =
224 snapshot.iter().find(|(name, _)| name == model)
225 else {
226 return Err((
227 "blocked_model_missing",
228 RuntimeError::UnknownModel(model.clone()),
229 ));
230 };
231 if dimensions.as_slice() != [*captured_dimensions] {
232 return Err((
233 "blocked_dimensions",
234 RuntimeError::InvalidInput("pending model dimensions conflict".into()),
235 ));
236 }
237 let (storage_name, live_dimensions) = self
238 .vector_model_metadata(model)
239 .map_err(|error| ("blocked_model_missing", error))?;
240 if storage_name != *model || live_dimensions != *captured_dimensions {
241 return Err((
242 "blocked_dimensions",
243 RuntimeError::InvalidInput("pending model dimensions changed".into()),
244 ));
245 }
246 let outcome = self
247 .embed_document_with_model_outcome_for_token(
248 token,
249 model,
250 &super::entity_fts_document(entity).body,
251 )
252 .await
253 .map_err(|error| ("embedding", error))?;
254 report.observe(&outcome);
255 if outcome.vector.len() != *captured_dimensions
256 || crate::atomic_message::non_finite_index(&outcome.vector).is_some()
257 {
258 return Err((
259 "invalid_vector",
260 RuntimeError::InvalidInput("invalid pending entity vector".into()),
261 ));
262 }
263 if self
266 .vector_model_metadata(model)
267 .map_err(|error| ("vector_prepare", error))?
268 != (model.clone(), *captured_dimensions)
269 {
270 return Err((
271 "blocked_dimensions",
272 RuntimeError::InvalidInput("pending model dimensions changed".into()),
273 ));
274 }
275 self.vectors_for_model(token, model)
276 .map_err(|error| ("vector_prepare", error))?;
277 let table = format!("vec_{}", crate::config::sanitize_key(model));
278 Ok(Self::entity_vector_insert_statements(
279 &table,
280 entity,
281 model,
282 &outcome.vector,
283 ))
284 }
285 }
286 }
287
288 pub(super) async fn publish_pending_entity_indexes(
289 &self,
290 token: &NamespaceToken,
291 entity: &Entity,
292 pending: &PendingEntityIndex,
293 snapshot: &[(String, usize)],
294 ) -> EntityUpdateReport {
295 let mut report = EntityUpdateReport::complete(EmbeddingTruncationReport::default(), true);
296 if entity.id != pending.entity_id
297 || entity.namespace != pending.namespace
298 || entity.version != pending.source_version
299 || entity.deleted_at.is_some()
300 || entity.merged_into.is_some()
301 {
302 for target in pending.targets.iter().filter(|target| !target.done) {
303 report.indexing.unfinished(&target.kind, "superseded");
304 }
305 return report;
306 }
307 let mut unsettled_attempt = false;
308 for target in &pending.targets {
309 if target.done {
310 continue;
311 }
312 if unsettled_attempt {
313 report
314 .indexing
315 .unfinished(&target.kind, "not_attempted_after_unknown");
316 continue;
317 }
318 if target.kind.is_blocked() {
319 report
320 .indexing
321 .unfinished(&target.kind, "blocked_dimensions");
322 continue;
323 }
324 if target.next_eligible_at > chrono::Utc::now().timestamp_micros() {
325 report.indexing.unfinished(&target.kind, "backoff");
326 continue;
327 }
328 let result = match self
329 .prepare_pending_entity_target(
330 token,
331 entity,
332 &target.kind,
333 snapshot,
334 &mut report.embedding,
335 )
336 .await
337 {
338 Ok(statements) => {
339 #[cfg(test)]
340 match &target.kind {
341 PendingIndexTargetKind::Fts => {
342 super::race_seam::pause_before_entity_index_publish().await
343 }
344 PendingIndexTargetKind::Vector { .. } => {
345 super::race_seam::pause_before_entity_vector_publish().await
346 }
347 }
348 self.publish_pending_entity_target(pending, &target.kind, statements)
349 .await
350 .map_err(|error| ("index_publish", error))
351 }
352 Err(error) => Err(error),
353 };
354 match result {
355 Ok(true) => {}
356 Ok(false) => report.indexing.unfinished(&target.kind, "superseded"),
357 Err((_, error)) if unsettled(&error) => {
358 unsettled_attempt = true;
361 report.indexing.unfinished(&target.kind, "outcome_unknown");
362 }
363 Err((stage, _)) => {
364 match self
365 .record_pending_entity_failure(pending, target, stage)
366 .await
367 {
368 Ok(true) => report.indexing.unfinished(&target.kind, stage),
369 Ok(false) => report.indexing.unfinished(&target.kind, "superseded"),
370 Err(error) => {
371 unsettled_attempt = unsettled(&error);
372 report.indexing.unfinished(
373 &target.kind,
374 if unsettled_attempt {
375 "failure_record_outcome_unknown"
376 } else {
377 "failure_record_failed"
378 },
379 );
380 }
381 }
382 }
383 }
384 }
385 report
386 }
387}
388
389#[cfg(test)]
390#[path = "entity_pending_index_tests.rs"]
391mod tests;