1use serde_json::Value;
39use uuid::Uuid;
40
41use khive_storage::note::Note;
42use khive_storage::types::SqlValue;
43use khive_storage::{SqlStatement, StorageCapability, StorageError};
44use khive_types::SubstrateKind;
45
46use crate::atomic_plan::{AddNotePlan, AffectedRowGuard, PlanStatement, PostCommitEffect};
47use crate::atomic_runner::{run_atomic_unit, AtomicOpPlan, AtomicRunOutcome};
48use crate::config::NamespaceToken;
49use crate::curation::note_fts_document;
50use crate::error::{RuntimeError, RuntimeResult};
51use crate::runtime::KhiveRuntime;
52
53pub struct AtomicNoteSpec<'a> {
59 pub token: &'a NamespaceToken,
61 pub id: Option<Uuid>,
66 pub kind: &'a str,
67 pub name: Option<&'a str>,
68 pub content: &'a str,
69 pub properties: Option<Value>,
70}
71
72fn f32_vec_to_bytes(data: &[f32]) -> Vec<u8> {
79 let mut bytes = Vec::with_capacity(data.len() * 4);
80 for f in data {
81 bytes.extend_from_slice(&f.to_ne_bytes());
82 }
83 bytes
84}
85
86fn non_finite_index(data: &[f32]) -> Option<usize> {
91 data.iter().position(|v| !v.is_finite())
92}
93
94fn non_finite_vector_error(idx: usize, value: f32) -> RuntimeError {
99 RuntimeError::Storage(StorageError::InvalidInput {
100 capability: StorageCapability::Vectors,
101 operation: "vec_insert".into(),
102 message: format!(
103 "non-finite value at index {idx}: {value} \
104 (NaN/Inf values corrupt distance computations)"
105 ),
106 })
107}
108
109#[cfg(any(test, feature = "fault-injection"))]
122fn injected_failure_statement(label: &str) -> PlanStatement {
123 PlanStatement {
124 statement: SqlStatement {
125 sql: "UPDATE notes SET updated_at = updated_at WHERE 1 = 0".to_string(),
126 params: vec![],
127 label: Some(label.to_string()),
128 },
129 guard: Some(AffectedRowGuard::exactly(1)),
130 }
131}
132
133#[cfg(any(test, feature = "fault-injection"))]
134fn maybe_inject_fts_failure(namespace: &str, label: &str) -> Option<PlanStatement> {
135 crate::operations::consume_fts_fail_fault(namespace).then(|| injected_failure_statement(label))
136}
137#[cfg(not(any(test, feature = "fault-injection")))]
138fn maybe_inject_fts_failure(_namespace: &str, _label: &str) -> Option<PlanStatement> {
139 None
140}
141
142#[cfg(any(test, feature = "fault-injection"))]
143fn maybe_inject_vector_failure(namespace: &str, label: &str) -> Option<PlanStatement> {
144 crate::operations::consume_vector_fail_fault(namespace)
145 .then(|| injected_failure_statement(label))
146}
147#[cfg(not(any(test, feature = "fault-injection")))]
148fn maybe_inject_vector_failure(_namespace: &str, _label: &str) -> Option<PlanStatement> {
149 None
150}
151
152#[allow(clippy::too_many_arguments)]
160fn vector_insert_statements(
161 table: &str,
162 namespace: &str,
163 subject_id: Uuid,
164 field: &str,
165 embedding_model: &str,
166 embedding: &[f32],
167 label_prefix: &str,
168) -> Vec<PlanStatement> {
169 let subject = subject_id.to_string();
170 let kind_str = SubstrateKind::Note.to_string();
171 let blob = f32_vec_to_bytes(embedding);
172 vec![
173 PlanStatement {
176 statement: SqlStatement {
177 sql: format!(
178 "INSERT INTO ann_write_log \
179 (namespace, embedding_model, kind, field, subject_id, op) \
180 SELECT namespace, embedding_model, kind, field, subject_id, 'delete' \
181 FROM {table} WHERE subject_id = ?1 AND NOT \
182 (namespace = ?2 AND embedding_model = ?3 AND kind = ?4 AND field = ?5)"
183 ),
184 params: vec![
185 SqlValue::Text(subject.clone()),
186 SqlValue::Text(namespace.to_string()),
187 SqlValue::Text(embedding_model.to_string()),
188 SqlValue::Text(kind_str.clone()),
189 SqlValue::Text(field.to_string()),
190 ],
191 label: Some(format!("{label_prefix}-log-delete")),
192 },
193 guard: None,
194 },
195 PlanStatement {
196 statement: SqlStatement {
197 sql: format!("DELETE FROM {table} WHERE subject_id = ?1"),
198 params: vec![SqlValue::Text(subject.clone())],
199 label: Some(format!("{label_prefix}-delete")),
200 },
201 guard: None,
202 },
203 PlanStatement {
204 statement: SqlStatement {
205 sql: format!(
206 "INSERT INTO {table} \
207 (subject_id, namespace, kind, field, embedding_model, embedding) \
208 VALUES (?1, ?2, ?3, ?4, ?5, ?6)"
209 ),
210 params: vec![
211 SqlValue::Text(subject.clone()),
212 SqlValue::Text(namespace.to_string()),
213 SqlValue::Text(kind_str.clone()),
214 SqlValue::Text(field.to_string()),
215 SqlValue::Text(embedding_model.to_string()),
216 SqlValue::Blob(blob),
217 ],
218 label: Some(format!("{label_prefix}-insert")),
219 },
220 guard: None,
221 },
222 PlanStatement {
223 statement: SqlStatement {
224 sql: "INSERT INTO ann_write_log \
225 (namespace, embedding_model, kind, field, subject_id, op) \
226 VALUES (?1, ?2, ?3, ?4, ?5, 'upsert')"
227 .to_string(),
228 params: vec![
229 SqlValue::Text(namespace.to_string()),
230 SqlValue::Text(embedding_model.to_string()),
231 SqlValue::Text(kind_str),
232 SqlValue::Text(field.to_string()),
233 SqlValue::Text(subject),
234 ],
235 label: Some(format!("{label_prefix}-log-upsert")),
236 },
237 guard: None,
238 },
239 ]
240}
241
242pub async fn create_notes_atomic(
249 runtime: &KhiveRuntime,
250 specs: Vec<AtomicNoteSpec<'_>>,
251) -> RuntimeResult<Vec<Note>> {
252 Ok(create_notes_atomic_with_report(runtime, specs).await?.0)
253}
254
255pub async fn create_notes_atomic_with_report(
259 runtime: &KhiveRuntime,
260 specs: Vec<AtomicNoteSpec<'_>>,
261) -> RuntimeResult<(Vec<Note>, crate::retrieval::EmbeddingTruncationReport)> {
262 let mut notes: Vec<Note> = Vec::with_capacity(specs.len());
265 for spec in &specs {
266 runtime.validate_note_kind(spec.kind)?;
267 crate::secret_gate::check(spec.content)?;
268 if let Some(n) = spec.name {
269 crate::secret_gate::check(n)?;
270 }
271 if let Some(ref p) = spec.properties {
272 crate::secret_gate::check_json(p)?;
273 }
274
275 let ns = spec.token.namespace().as_str();
276 let mut note = Note::new(ns, spec.kind, spec.content);
277 if let Some(id) = spec.id {
278 note.id = id;
279 }
280 if let Some(n) = spec.name {
281 note = note.with_name(n);
282 }
283 if let Some(p) = spec.properties.clone() {
284 note = note.with_properties(p);
285 }
286 notes.push(note);
287 }
288
289 let embed_model_names = runtime.registered_embedding_model_names();
292 let mut embeddings: Vec<Vec<Option<Vec<f32>>>> = notes
294 .iter()
295 .map(|_| vec![None; embed_model_names.len()])
296 .collect();
297 let mut embedding_truncation = crate::retrieval::EmbeddingTruncationReport::default();
298
299 if !embed_model_names.is_empty() {
300 for model_name in &embed_model_names {
304 runtime.vectors_for_model(specs[0].token, model_name)?;
305 }
306
307 let usage_ctx = crate::usage::current();
308 let mut join_set = tokio::task::JoinSet::new();
309 for (note_idx, (spec, note)) in specs.iter().zip(notes.iter()).enumerate() {
310 let text = crate::curation::note_embedding_text(note);
311 for (model_idx, model_name) in embed_model_names.iter().enumerate() {
312 let rt = runtime.clone();
313 let token = spec.token.clone();
314 let name = model_name.clone();
315 let text = text.clone();
316 let ctx = usage_ctx.clone();
317 join_set.spawn(async move {
318 let fut = rt.embed_document_with_model_outcome_for_token(&token, &name, &text);
319 let result = match ctx {
320 Some(ctx) => crate::usage::scope(ctx, fut).await,
321 None => fut.await,
322 };
323 (note_idx, model_idx, result)
324 });
325 }
326 }
327
328 while let Some(joined) = join_set.join_next().await {
329 match joined {
330 Ok((note_idx, model_idx, Ok(outcome))) => {
331 embedding_truncation.observe(&outcome);
332 if outcome.truncated {
333 tracing::warn!(
334 model = %outcome.model_name,
335 source_bytes = outcome.source_bytes,
336 embedded_bytes = outcome.embedded_bytes,
337 "atomic note embedding input truncated; full content will be stored unchanged"
338 );
339 }
340 embeddings[note_idx][model_idx] = Some(outcome.vector);
341 }
342 Ok((_, _, Err(e))) => {
343 join_set.abort_all();
344 return Err(e);
345 }
346 Err(join_err) => {
347 join_set.abort_all();
348 return Err(RuntimeError::Internal(format!(
349 "embed task panicked: {join_err}"
350 )));
351 }
352 }
353 }
354 }
355
356 for embeddings_for_note in &embeddings {
362 for embedding in embeddings_for_note.iter().flatten() {
363 if let Some(idx) = non_finite_index(embedding) {
364 return Err(non_finite_vector_error(idx, embedding[idx]));
365 }
366 }
367 }
368
369 let mut plans: Vec<AtomicOpPlan> = Vec::with_capacity(notes.len());
372 for (note, embeddings_for_note) in notes.iter().zip(embeddings) {
373 let mut statements = vec![PlanStatement {
374 statement: khive_db::stores::note::note_upsert_statement(note),
375 guard: Some(AffectedRowGuard::exactly(1)),
376 }];
377
378 if let Some(fault) = maybe_inject_fts_failure(¬e.namespace, "fault-injected-fts") {
379 statements.push(fault);
380 } else {
381 statements.push(PlanStatement {
387 statement: khive_db::stores::text::delete_document_statement(
388 "fts_notes",
389 ¬e.namespace,
390 note.id,
391 ),
392 guard: None,
393 });
394 statements.push(PlanStatement {
395 statement: khive_db::stores::text::insert_document_statement(
396 "fts_notes",
397 ¬e_fts_document(note),
398 ),
399 guard: None,
400 });
401 }
402
403 if let Some(fault) = maybe_inject_vector_failure(¬e.namespace, "fault-injected-vector") {
404 statements.push(fault);
405 } else {
406 for (model_name, embedding) in embed_model_names.iter().zip(embeddings_for_note) {
407 let embedding = embedding.expect("every model index observed exactly once");
408 let table = format!("vec_{}", crate::config::sanitize_key(model_name));
409 statements.extend(vector_insert_statements(
410 &table,
411 ¬e.namespace,
412 note.id,
413 "note.content",
414 model_name,
415 &embedding,
416 &format!("atomic-message-vec-{table}-{}", note.id),
417 ));
418 }
419 }
420
421 plans.push(AtomicOpPlan::AddNote(AddNotePlan {
422 note_id: note.id,
423 statements,
424 post_commit: PostCommitEffect::None,
425 }));
426 }
427
428 match run_atomic_unit(runtime.sql().as_ref(), plans).await {
430 Ok(AtomicRunOutcome::Committed { .. }) => Ok((notes, embedding_truncation)),
431 Ok(AtomicRunOutcome::RolledBack {
432 failed_op_index,
433 failure,
434 }) => Err(RuntimeError::Internal(format!(
435 "atomic multi-note write rolled back at op {failed_op_index}: {failure:?}"
436 ))),
437 Err(e) => Err(RuntimeError::Storage(e.0)),
438 }
439}
440
441#[cfg(test)]
442mod tests {
443 use super::*;
444 use async_trait::async_trait;
445 use lattice_embed::{EmbedError, EmbeddingModel, EmbeddingService, MAX_TEXT_CHARS};
446
447 use khive_types::Namespace;
448
449 use crate::embedder_registry::EmbedderProvider;
450
451 const NAN_MODEL: &str = "atomic-message-nan-model";
452 const NAN_DIMS: usize = 4;
453
454 struct NanService;
455 #[async_trait]
456 impl EmbeddingService for NanService {
457 async fn embed(
458 &self,
459 texts: &[String],
460 _model: EmbeddingModel,
461 ) -> Result<Vec<Vec<f32>>, EmbedError> {
462 Ok(texts.iter().map(|_| vec![f32::NAN; NAN_DIMS]).collect())
463 }
464 fn supports_model(&self, _model: EmbeddingModel) -> bool {
465 true
466 }
467 fn name(&self) -> &'static str {
468 NAN_MODEL
469 }
470 }
471 struct NanProvider;
472 #[async_trait]
473 impl EmbedderProvider for NanProvider {
474 fn name(&self) -> &str {
475 NAN_MODEL
476 }
477 fn dimensions(&self) -> usize {
478 NAN_DIMS
479 }
480 async fn build(&self) -> RuntimeResult<std::sync::Arc<dyn EmbeddingService>> {
481 Ok(std::sync::Arc::new(NanService))
482 }
483 }
484
485 struct TruncationService;
486 #[async_trait]
487 impl EmbeddingService for TruncationService {
488 async fn embed(
489 &self,
490 texts: &[String],
491 _model: EmbeddingModel,
492 ) -> Result<Vec<Vec<f32>>, EmbedError> {
493 Ok(texts
494 .iter()
495 .map(|_| vec![0.5; EmbeddingModel::MultilingualE5Base.dimensions()])
496 .collect())
497 }
498 fn supports_model(&self, _model: EmbeddingModel) -> bool {
499 true
500 }
501 fn name(&self) -> &'static str {
502 "multilingual-e5-base"
503 }
504 }
505
506 struct TruncationProvider;
507 #[async_trait]
508 impl EmbedderProvider for TruncationProvider {
509 fn name(&self) -> &str {
510 "multilingual-e5-base"
511 }
512 fn dimensions(&self) -> usize {
513 EmbeddingModel::MultilingualE5Base.dimensions()
514 }
515 async fn build(&self) -> RuntimeResult<std::sync::Arc<dyn EmbeddingService>> {
516 Ok(std::sync::Arc::new(TruncationService))
517 }
518 }
519
520 async fn fts_row_count(runtime: &KhiveRuntime, namespace: &str) -> i64 {
521 let mut reader = runtime.sql().reader().await.expect("sql reader");
522 match reader
523 .query_scalar(SqlStatement {
524 sql: "SELECT COUNT(*) FROM fts_notes WHERE namespace = ?1".to_string(),
525 params: vec![SqlValue::Text(namespace.to_string())],
526 label: None,
527 })
528 .await
529 .expect("fts count query")
530 {
531 Some(SqlValue::Integer(n)) => n,
532 other => panic!("unexpected fts count result: {other:?}"),
533 }
534 }
535
536 async fn ann_write_log_count(runtime: &KhiveRuntime, namespace: &str, model: &str) -> i64 {
537 let mut reader = runtime.sql().reader().await.expect("sql reader");
538 match reader
539 .query_scalar(SqlStatement {
540 sql: "SELECT COUNT(*) FROM ann_write_log \
541 WHERE namespace = ?1 AND embedding_model = ?2"
542 .to_string(),
543 params: vec![
544 SqlValue::Text(namespace.to_string()),
545 SqlValue::Text(model.to_string()),
546 ],
547 label: None,
548 })
549 .await
550 .expect("ann_write_log count query")
551 {
552 Some(SqlValue::Integer(n)) => n,
553 other => panic!("unexpected ann_write_log count result: {other:?}"),
554 }
555 }
556
557 #[tokio::test]
562 async fn create_notes_atomic_rejects_non_finite_embedding_before_any_write() {
563 let runtime = KhiveRuntime::memory().expect("in-memory runtime");
564 runtime.register_embedder(NanProvider);
565 let ns = "atomic-message-nan-test";
566 let token = runtime
567 .authorize(Namespace::parse(ns).unwrap())
568 .expect("authorize");
569
570 let result = create_notes_atomic(
571 &runtime,
572 vec![AtomicNoteSpec {
573 token: &token,
574 id: None,
575 kind: "observation",
576 name: None,
577 content: "nan embedding content",
578 properties: None,
579 }],
580 )
581 .await;
582
583 assert!(
584 result.is_err(),
585 "a non-finite embedding must be rejected; got {result:?}"
586 );
587
588 let alive = runtime
589 .list_notes(&token, Some("observation"), 100, 0)
590 .await
591 .expect("list_notes")
592 .into_iter()
593 .filter(|n| n.deleted_at.is_none())
594 .count();
595 assert_eq!(
596 alive, 0,
597 "no note row may be committed when an embedding is non-finite"
598 );
599
600 assert_eq!(
601 fts_row_count(&runtime, ns).await,
602 0,
603 "no FTS document may be committed when an embedding is non-finite"
604 );
605
606 let vs = runtime
607 .vectors_for_model(&token, NAN_MODEL)
608 .expect("vec store");
609 assert_eq!(
610 vs.count().await.expect("count"),
611 0,
612 "no vector row may be committed when an embedding is non-finite"
613 );
614
615 assert_eq!(
616 ann_write_log_count(&runtime, ns, NAN_MODEL).await,
617 0,
618 "no ann_write_log row may be committed when an embedding is non-finite"
619 );
620 }
621
622 #[tokio::test]
623 async fn create_notes_atomic_with_report_preserves_truncation_outcome() {
624 let runtime = KhiveRuntime::memory().expect("in-memory runtime");
625 runtime.register_embedder(TruncationProvider);
626 let token = runtime
627 .authorize(Namespace::parse("atomic-message-truncation-test").unwrap())
628 .expect("authorize");
629 let content = "x".repeat(MAX_TEXT_CHARS);
630
631 let (notes, report) = create_notes_atomic_with_report(
632 &runtime,
633 vec![AtomicNoteSpec {
634 token: &token,
635 id: None,
636 kind: "observation",
637 name: None,
638 content: &content,
639 properties: None,
640 }],
641 )
642 .await
643 .expect("atomic note write");
644
645 assert_eq!(notes.len(), 1);
646 assert_eq!(report.truncated, 1);
647 assert_eq!(
648 report.discarded_bytes,
649 "passage: ".len() as u64,
650 "E5 document-prefix reservation must be reflected in the returned report"
651 );
652 }
653
654 #[tokio::test]
659 async fn create_notes_atomic_upserts_fts_document_for_reused_note_id() {
660 let runtime = KhiveRuntime::memory().expect("in-memory runtime");
661 let ns = "atomic-message-fts-upsert-test";
662 let token = runtime
663 .authorize(Namespace::parse(ns).unwrap())
664 .expect("authorize");
665 let id = Uuid::new_v4();
666
667 create_notes_atomic(
668 &runtime,
669 vec![AtomicNoteSpec {
670 token: &token,
671 id: Some(id),
672 kind: "observation",
673 name: None,
674 content: "first content",
675 properties: None,
676 }],
677 )
678 .await
679 .expect("first write with supplied id");
680
681 create_notes_atomic(
682 &runtime,
683 vec![AtomicNoteSpec {
684 token: &token,
685 id: Some(id),
686 kind: "observation",
687 name: None,
688 content: "second content replacing the first",
689 properties: None,
690 }],
691 )
692 .await
693 .expect("second write reusing the same id");
694
695 assert_eq!(
696 fts_row_count(&runtime, ns).await,
697 1,
698 "exactly one FTS document may exist for the reused id"
699 );
700
701 let text = runtime.text_for_notes(&token).expect("text store");
702 let doc = text
703 .get_document(ns, id)
704 .await
705 .expect("get_document")
706 .expect("document exists for the reused id");
707 assert!(
708 doc.body.contains("second content"),
709 "the surviving FTS document must reflect the latest content; got {:?}",
710 doc.body
711 );
712 }
713}