Skip to main content

khive_runtime/curation/
entity_pending_index.rs

1//! Durable publication protocol for deferred entity updates (ADR-014 Amendment 7).
2
3use std::any::Any;
4
5use khive_db::pending_index::{
6    self, PendingEntityIndex, PendingIndexTarget, PendingIndexTargetKind,
7};
8use khive_storage::{AtomicUnitOp, Entity, SqlStatement, StorageError, WriterTaskRequestState};
9use serde::Serialize;
10
11use crate::retrieval::EmbeddingTruncationReport;
12use crate::{KhiveRuntime, NamespaceToken, PostCommitDegradation, RuntimeError, RuntimeResult};
13
14#[cfg(test)]
15tokio::task_local! {
16    static UNKNOWN_AFTER_PUBLISH: bool;
17}
18
19/// One required index whose publication was not established by this update.
20#[derive(Clone, Debug, Serialize)]
21pub struct EntityIndexFailure {
22    pub target: PendingIndexTargetKind,
23    /// Sanitized stage; never entity content or provider error text.
24    pub stage: String,
25}
26
27/// The indexing outcome of the committed entity revision, not a backlog census.
28#[derive(Clone, Debug, Serialize)]
29pub struct EntityIndexReport {
30    pub status: &'static str,
31    pub unfinished: Vec<EntityIndexFailure>,
32}
33
34impl EntityIndexReport {
35    pub(super) fn complete(required: bool) -> Self {
36        Self {
37            status: if required {
38                "complete"
39            } else {
40                "not_requested"
41            },
42            unfinished: Vec::new(),
43        }
44    }
45
46    fn unfinished(&mut self, target: &PendingIndexTargetKind, stage: &str) {
47        self.status = "unfinished";
48        self.unfinished.push(EntityIndexFailure {
49            target: target.clone(),
50            stage: stage.into(),
51        });
52    }
53}
54
55/// Report-aware updates retain their committed entity even when indexes fail.
56#[derive(Clone, Debug, Serialize)]
57pub struct EntityUpdateReport {
58    pub embedding: EmbeddingTruncationReport,
59    pub indexing: EntityIndexReport,
60    pub post_commit_degradations: Vec<PostCommitDegradation>,
61}
62
63impl EntityUpdateReport {
64    pub(super) fn complete(embedding: EmbeddingTruncationReport, required: bool) -> Self {
65        Self {
66            embedding,
67            indexing: EntityIndexReport::complete(required),
68            post_commit_degradations: Vec::new(),
69        }
70    }
71
72    /// Compatibility callers cannot silently discard committed-but-unfinished work.
73    pub(super) fn into_legacy(
74        self,
75        entity: Entity,
76    ) -> RuntimeResult<(Entity, EmbeddingTruncationReport)> {
77        let mut degradations = self.post_commit_degradations;
78        degradations.extend(self.indexing.unfinished.into_iter().map(|failure| {
79            PostCommitDegradation {
80                stage: "entity_index_unfinished",
81                error: serde_json::json!(failure).to_string(),
82            }
83        }));
84        crate::operations::legacy_post_commit_result(
85            "update_entity",
86            entity.id,
87            (entity, self.embedding),
88            degradations,
89        )
90    }
91}
92
93fn unsettled(error: &RuntimeError) -> bool {
94    error
95        .writer_task_failure_context()
96        .is_some_and(|context| context.request_state == WriterTaskRequestState::SideEffectsUnknown)
97        || matches!(error.refusal_source(), RuntimeError::Storage(error)
98        if error.sqlite_write_failure().is_some_and(|failure| failure.settlement_unknown))
99}
100
101impl KhiveRuntime {
102    /// The sole pending-target consumer, shared by inline work and future recovery.
103    /// The closure performs only synchronous-ready SQL under one writer transaction.
104    pub(super) async fn publish_pending_entity_target(
105        &self,
106        observed: &PendingEntityIndex,
107        target: &PendingIndexTargetKind,
108        statements: Vec<SqlStatement>,
109    ) -> RuntimeResult<bool> {
110        let observed = observed.clone();
111        let target = target.clone();
112        let op: AtomicUnitOp = Box::new(move |writer| {
113            Box::pin(async move {
114                let Some(current) = pending_index::read(writer, observed.entity_id).await? else {
115                    return Ok(Box::new(false) as Box<dyn Any + Send>);
116                };
117                if current.generation != observed.generation
118                    || current.namespace != observed.namespace
119                    || current.source_version != observed.source_version
120                    || !current.targets.iter().any(|entry| {
121                        entry.kind == target && !entry.done && !entry.kind.is_blocked()
122                    })
123                    || !pending_index::live_matches(writer, &current).await?
124                {
125                    return Ok(Box::new(false) as Box<dyn Any + Send>);
126                }
127                for statement in statements {
128                    writer.execute(statement).await?;
129                }
130                if !pending_index::finish(writer, current.entity_id, current.generation, &target)
131                    .await?
132                {
133                    // Never commit index writes without consuming the matching target.
134                    return Err(StorageError::Internal(
135                        "pending index consumption guard failed".into(),
136                    ));
137                }
138                Ok(Box::new(true) as Box<dyn Any + Send>)
139            })
140        });
141        let outcome = self.sql().atomic_unit(op).await?;
142        #[cfg(test)]
143        if UNKNOWN_AFTER_PUBLISH
144            .try_with(|enabled| *enabled)
145            .unwrap_or(false)
146        {
147            return Err(RuntimeError::Storage(StorageError::writer_task_terminated(
148                WriterTaskRequestState::SideEffectsUnknown,
149            )));
150        }
151        outcome
152            .downcast::<bool>()
153            .map(|result| *result)
154            .map_err(|_| RuntimeError::Internal("invalid pending index publication outcome".into()))
155    }
156
157    async fn record_pending_entity_failure(
158        &self,
159        observed: &PendingEntityIndex,
160        target: &PendingIndexTarget,
161        category: &'static str,
162    ) -> RuntimeResult<bool> {
163        let id = observed.entity_id;
164        let generation = observed.generation;
165        let kind = target.kind.clone();
166        // Slice 1 schedules no recovery. Preserve any existing delay; the later
167        // supervised recovery component owns its bounded backoff calculation.
168        let next_eligible_at = target.next_eligible_at;
169        let op: AtomicUnitOp = Box::new(move |writer| {
170            Box::pin(async move {
171                let recorded = pending_index::record_failure(
172                    writer,
173                    id,
174                    generation,
175                    &kind,
176                    category,
177                    next_eligible_at,
178                )
179                .await?;
180                Ok(Box::new(recorded) as Box<dyn Any + Send>)
181            })
182        });
183        self.sql()
184            .atomic_unit(op)
185            .await?
186            .downcast::<bool>()
187            .map(|result| *result)
188            .map_err(|_| RuntimeError::Internal("invalid pending index failure outcome".into()))
189    }
190
191    async fn prepare_pending_entity_target(
192        &self,
193        token: &NamespaceToken,
194        entity: &Entity,
195        target: &PendingIndexTargetKind,
196        snapshot: &[(String, usize)],
197        report: &mut EmbeddingTruncationReport,
198    ) -> Result<Vec<SqlStatement>, (&'static str, RuntimeError)> {
199        match target {
200            PendingIndexTargetKind::Fts => {
201                #[cfg(test)]
202                if crate::operations::consume_fts_fail_fault(&entity.namespace) {
203                    return Err((
204                        "fts_prepare",
205                        RuntimeError::Internal("injected FTS failure".into()),
206                    ));
207                }
208                self.text(token).map_err(|error| ("fts_prepare", error))?;
209                let doc = super::entity_fts_document(entity);
210                Ok(khive_db::stores::text::delete_document_statements(
211                    "fts_entities",
212                    &entity.namespace,
213                    entity.id,
214                )
215                .into_iter()
216                .chain(khive_db::stores::text::insert_document_statements(
217                    "fts_entities",
218                    &doc,
219                ))
220                .collect())
221            }
222            PendingIndexTargetKind::Vector { model, dimensions } => {
223                let Some((_, captured_dimensions)) =
224                    snapshot.iter().find(|(name, _)| name == model)
225                else {
226                    return Err((
227                        "blocked_model_missing",
228                        RuntimeError::UnknownModel(model.clone()),
229                    ));
230                };
231                if dimensions.as_slice() != [*captured_dimensions] {
232                    return Err((
233                        "blocked_dimensions",
234                        RuntimeError::InvalidInput("pending model dimensions conflict".into()),
235                    ));
236                }
237                let (storage_name, live_dimensions) = self
238                    .vector_model_metadata(model)
239                    .map_err(|error| ("blocked_model_missing", error))?;
240                if storage_name != *model || live_dimensions != *captured_dimensions {
241                    return Err((
242                        "blocked_dimensions",
243                        RuntimeError::InvalidInput("pending model dimensions changed".into()),
244                    ));
245                }
246                let outcome = self
247                    .embed_document_with_model_outcome_for_token(
248                        token,
249                        model,
250                        &super::entity_fts_document(entity).body,
251                    )
252                    .await
253                    .map_err(|error| ("embedding", error))?;
254                report.observe(&outcome);
255                if outcome.vector.len() != *captured_dimensions
256                    || crate::atomic_message::non_finite_index(&outcome.vector).is_some()
257                {
258                    return Err((
259                        "invalid_vector",
260                        RuntimeError::InvalidInput("invalid pending entity vector".into()),
261                    ));
262                }
263                // Provider resolution pins the serving registry entry. Recheck after
264                // it resolves so a concurrent setup replacement cannot change shape.
265                if self
266                    .vector_model_metadata(model)
267                    .map_err(|error| ("vector_prepare", error))?
268                    != (model.clone(), *captured_dimensions)
269                {
270                    return Err((
271                        "blocked_dimensions",
272                        RuntimeError::InvalidInput("pending model dimensions changed".into()),
273                    ));
274                }
275                self.vectors_for_model(token, model)
276                    .map_err(|error| ("vector_prepare", error))?;
277                let table = format!("vec_{}", crate::config::sanitize_key(model));
278                Ok(Self::entity_vector_insert_statements(
279                    &table,
280                    entity,
281                    model,
282                    &outcome.vector,
283                ))
284            }
285        }
286    }
287
288    pub(super) async fn publish_pending_entity_indexes(
289        &self,
290        token: &NamespaceToken,
291        entity: &Entity,
292        pending: &PendingEntityIndex,
293        snapshot: &[(String, usize)],
294    ) -> EntityUpdateReport {
295        let mut report = EntityUpdateReport::complete(EmbeddingTruncationReport::default(), true);
296        if entity.id != pending.entity_id
297            || entity.namespace != pending.namespace
298            || entity.version != pending.source_version
299            || entity.deleted_at.is_some()
300            || entity.merged_into.is_some()
301        {
302            for target in pending.targets.iter().filter(|target| !target.done) {
303                report.indexing.unfinished(&target.kind, "superseded");
304            }
305            return report;
306        }
307        let mut unsettled_attempt = false;
308        for target in &pending.targets {
309            if target.done {
310                continue;
311            }
312            if unsettled_attempt {
313                report
314                    .indexing
315                    .unfinished(&target.kind, "not_attempted_after_unknown");
316                continue;
317            }
318            if target.kind.is_blocked() {
319                report
320                    .indexing
321                    .unfinished(&target.kind, "blocked_dimensions");
322                continue;
323            }
324            if target.next_eligible_at > chrono::Utc::now().timestamp_micros() {
325                report.indexing.unfinished(&target.kind, "backoff");
326                continue;
327            }
328            let result = match self
329                .prepare_pending_entity_target(
330                    token,
331                    entity,
332                    &target.kind,
333                    snapshot,
334                    &mut report.embedding,
335                )
336                .await
337            {
338                Ok(statements) => {
339                    #[cfg(test)]
340                    match &target.kind {
341                        PendingIndexTargetKind::Fts => {
342                            super::race_seam::pause_before_entity_index_publish().await
343                        }
344                        PendingIndexTargetKind::Vector { .. } => {
345                            super::race_seam::pause_before_entity_vector_publish().await
346                        }
347                    }
348                    self.publish_pending_entity_target(pending, &target.kind, statements)
349                        .await
350                        .map_err(|error| ("index_publish", error))
351                }
352                Err(error) => Err(error),
353            };
354            match result {
355                Ok(true) => {}
356                Ok(false) => report.indexing.unfinished(&target.kind, "superseded"),
357                Err((_, error)) if unsettled(&error) => {
358                    // No reread, retry metadata write, or absence inference settles
359                    // a writer whose acknowledgement was lost.
360                    unsettled_attempt = true;
361                    report.indexing.unfinished(&target.kind, "outcome_unknown");
362                }
363                Err((stage, _)) => {
364                    match self
365                        .record_pending_entity_failure(pending, target, stage)
366                        .await
367                    {
368                        Ok(true) => report.indexing.unfinished(&target.kind, stage),
369                        Ok(false) => report.indexing.unfinished(&target.kind, "superseded"),
370                        Err(error) => {
371                            unsettled_attempt = unsettled(&error);
372                            report.indexing.unfinished(
373                                &target.kind,
374                                if unsettled_attempt {
375                                    "failure_record_outcome_unknown"
376                                } else {
377                                    "failure_record_failed"
378                                },
379                            );
380                        }
381                    }
382                }
383            }
384        }
385        report
386    }
387}
388
389#[cfg(test)]
390#[path = "entity_pending_index_tests.rs"]
391mod tests;