Skip to main content

khive_runtime/
atomic_message.rs

1//! Atomic multi-note write primitive: commits a set of notes — each with its
2//! FTS document and its kind-selected embedding model rows — in ONE
3//! writer transaction, instead of one `create_note` call per note.
4//!
5//! Built for `khive-pack-comm`'s `dual_write_message` (outbound + inbound
6//! copy of a `comm.send`/`comm.reply`), which previously cost roughly a
7//! dozen separate writer acquisitions per send: two `create_note_inner`
8//! calls (row + FTS + one vector insert per registered model each) plus a
9//! root-send `thread_id` patch. Shaped as `create_notes_atomic_with_report(Vec<AtomicNoteSpec>)`
10//! rather than a comm-specific pair primitive so other multi-write verbs
11//! can share the same preparation. Keyed `memory.remember` appends a required
12//! annotation and final key claim before committing its prepared plan.
13//!
14//! # Embed-first
15//!
16//! Embedding is slow compute (network/model calls). Every distinct content's
17//! embeddings, across the selected models, are computed **before** any
18//! transaction opens — the writer is held only for synchronous DML, exactly
19//! like the rest of the ADR-099 atomic-unit machinery
20//! (`atomic_plan`/`atomic_runner`).
21//! This is the same reason `atomic_prepare::prepare_add_note` defers vector
22//! indexing to a post-commit `PostCommitEffect::ReindexNote`; the difference
23//! here is embeddings are computed *before* commit instead of *after*, so
24//! the vector rows land in the SAME atomic unit as the note row and FTS
25//! document, and no post-commit reindex is needed at all — every plan built
26//! by [`create_notes_atomic_with_report`] carries `post_commit: PostCommitEffect::None`.
27//!
28//! # One writer acquisition
29//!
30//! Each note becomes its own [`AddNotePlan`]; every spec's plan is applied by
31//! ONE [`crate::atomic_runner::run_atomic_unit`] commit pass — one [`khive_storage::SqlAccess::atomic_unit`],
32//! one writer checkout, one WAL commit for the whole set. A failure on any
33//! note's plan rolls back the ENTIRE unit (`atomic_runner`'s documented
34//! guarantee: a later op's failure unwinds even an earlier op's own
35//! already-`RELEASE`d `SAVEPOINT`), so a crash or guard failure partway
36//! through never leaves an orphan copy — the process-crash gap the two-call
37//! version of `dual_write_message` used to document is closed by
38//! construction.
39
40use std::collections::HashMap;
41
42use serde_json::Value;
43use uuid::Uuid;
44
45use khive_storage::note::Note;
46use khive_storage::types::SqlValue;
47use khive_storage::{SqlStatement, StorageCapability, StorageError};
48use khive_types::SubstrateKind;
49
50use crate::atomic_plan::{AddNotePlan, AffectedRowGuard, PlanStatement, PostCommitEffect};
51use crate::atomic_runner::{AtomicOpPlan, AtomicRunOutcome};
52use crate::config::NamespaceToken;
53use crate::curation::note_fts_document;
54use crate::error::{RuntimeError, RuntimeResult};
55use crate::runtime::KhiveRuntime;
56
57/// One note to write as part of an atomic set. Mirrors the subset of
58/// `create_note_inner`'s parameters `comm.send`/`comm.reply` actually use —
59/// no `annotates`, `salience`, `decay_factor`, `embedding_content` override,
60/// or explicit `embedding_model` pin. Keyed-memory preparation carries those
61/// through internal options without changing the public multi-note spec.
62pub struct AtomicNoteSpec<'a> {
63    /// Namespace + actor identity this note is written under.
64    pub token: &'a NamespaceToken,
65    /// Caller-supplied id, when the id must be known before the write (e.g.
66    /// a root send's own id becomes the canonical `thread_id` stored in
67    /// BOTH copies' properties). `None` generates a fresh id, same as
68    /// `Note::new`.
69    pub id: Option<Uuid>,
70    pub kind: &'a str,
71    pub name: Option<&'a str>,
72    pub content: &'a str,
73    pub properties: Option<Value>,
74}
75
76#[derive(Clone, Copy, Default)]
77pub(crate) struct AtomicNoteOptions<'a> {
78    pub salience: Option<f64>,
79    pub decay_factor: Option<f64>,
80    pub embedding_model: Option<&'a str>,
81    pub embedding_content: Option<&'a str>,
82    pub embed: Option<bool>,
83    pub key: Option<&'a str>,
84    /// Keyed memory only: persist a zero-model receipt header and each model's
85    /// exact log-upsert rowid in this same atomic unit.
86    pub memory_visibility_receipt: bool,
87    /// Ask the writer transaction to compare a live key holder against this
88    /// candidate and report the result (ADR-172 Amendment 6). Every other
89    /// keyed route defaults this to `false` and gets today's Details shape
90    /// on a key conflict, with no comparison outcome in it at all.
91    pub replay_receipt: bool,
92    pub fence: Option<&'a crate::note_write::NoteFences>,
93    /// `spec.properties` is already the fully validator-derived value, so
94    /// skip calling the note-write validator again below (ADR-172 Amendment
95    /// 6's keyed singleton-create staging derives properties once, before
96    /// its initial holder check, and reuses that same value through
97    /// preparation and any final-check fallback; the installed validator is
98    /// not guaranteed idempotent, and a kg regression test counts its calls,
99    /// so deriving twice for one request is an observably different
100    /// candidate, not a redundant no-op). Every other caller leaves this
101    /// `false` and keeps deriving from raw caller-supplied properties here.
102    pub properties_already_derived: bool,
103}
104
105pub(crate) struct AtomicNoteRequest<'a> {
106    pub spec: AtomicNoteSpec<'a>,
107    pub options: AtomicNoteOptions<'a>,
108}
109
110pub(crate) struct PreparedAtomicNotes {
111    pub notes: Vec<Note>,
112    pub plans: Vec<AtomicOpPlan>,
113    pub embedding_truncation: crate::retrieval::EmbeddingTruncationReport,
114}
115
116/// Mirrors `khive-db`'s private `non_finite_index` — this crate has no
117/// visibility into that helper, so the check is reproduced here to keep the
118/// atomic path's embedding validation observably identical to the canonical
119/// `create_note_inner` -> `VectorStore::insert` path.
120pub(crate) fn non_finite_index(data: &[f32]) -> Option<usize> {
121    data.iter().position(|v| !v.is_finite())
122}
123
124/// Same error shape (capability, operation label, message) as `khive-db`'s
125/// private `non_finite_vector_error("vec_insert", ..)` — the atomic path
126/// must reject a non-finite embedding exactly as the raw `VectorStore::insert`
127/// DML does, not silently write it.
128pub(crate) fn non_finite_vector_error(idx: usize, value: f32) -> RuntimeError {
129    RuntimeError::Storage(StorageError::InvalidInput {
130        capability: StorageCapability::Vectors,
131        operation: "vec_insert".into(),
132        message: format!(
133            "non-finite value at index {idx}: {value} \
134             (NaN/Inf values corrupt distance computations)"
135        ),
136    })
137}
138
139/// A harmless no-op `UPDATE` with a statically zero-row predicate (`WHERE 1
140/// = 0`, not a match against a specific id), guarded `exactly(1)` so it
141/// always fails its guard inside the atomic unit regardless of which note
142/// ids exist in the store — a fixed nil-UUID predicate would pass its guard
143/// if a caller ever supplied that id. Spliced in by test-only fault
144/// injection (see [`maybe_inject_fts_failure`]/[`maybe_inject_vector_failure`])
145/// to prove the whole unit — not just the failing note's own plan — rolls
146/// back.
147///
148/// `fault-injection` is an ordinary Cargo feature (see this crate's
149/// `Cargo.toml`); it must never be enabled in a packaged/release build —
150/// doing so would compile this fault-injection path into a shipped binary.
151#[cfg(any(test, feature = "fault-injection"))]
152fn injected_failure_statement(label: &str) -> PlanStatement {
153    PlanStatement {
154        statement: SqlStatement {
155            sql: "UPDATE notes SET updated_at = updated_at WHERE 1 = 0".to_string(),
156            params: vec![],
157            label: Some(label.to_string()),
158        },
159        guard: Some(AffectedRowGuard::exactly(1)),
160    }
161}
162
163#[cfg(any(test, feature = "fault-injection"))]
164fn maybe_inject_fts_failure(namespace: &str, label: &str) -> Option<PlanStatement> {
165    crate::operations::consume_fts_fail_fault(namespace).then(|| injected_failure_statement(label))
166}
167#[cfg(not(any(test, feature = "fault-injection")))]
168fn maybe_inject_fts_failure(_namespace: &str, _label: &str) -> Option<PlanStatement> {
169    None
170}
171
172#[cfg(any(test, feature = "fault-injection"))]
173fn maybe_inject_vector_failure(namespace: &str, label: &str) -> Option<PlanStatement> {
174    crate::operations::consume_vector_fail_fault(namespace)
175        .then(|| injected_failure_statement(label))
176}
177#[cfg(not(any(test, feature = "fault-injection")))]
178fn maybe_inject_vector_failure(_namespace: &str, _label: &str) -> Option<PlanStatement> {
179    None
180}
181
182/// DELETE-then-INSERT-then-log statements for one (note, model) vector row.
183/// This static atomic plan cannot branch on an identity-constrained DELETE's
184/// affected-row count, so it retains the general delete-log scan used before
185/// `khive-db::stores::vectors::replace_vector_row_dml` gained its live-
186/// connection fast path. It uses the same plain-[`PlanStatement`] technique
187/// as `atomic_prepare::push_index_purge_statements`. Unguarded: same
188/// convention as the FTS-insert statement in
189/// `atomic_prepare::prepare_add_note` (the row's existence guard is carried by
190/// the plan's primary note-row statement, applied first).
191#[allow(clippy::too_many_arguments)]
192pub(crate) fn vector_insert_statements(
193    table: &str,
194    namespace: &str,
195    subject_id: Uuid,
196    field: &str,
197    embedding_model: &str,
198    embedding: &[f32],
199    label_prefix: &str,
200) -> Vec<PlanStatement> {
201    vector_insert_statements_for_substrate(
202        table,
203        namespace,
204        subject_id,
205        SubstrateKind::Note,
206        field,
207        embedding_model,
208        embedding,
209        label_prefix,
210        None,
211    )
212}
213
214/// Build atomic vector and ANN-log writes for a typed substrate. The caller
215/// chooses whether the two single-row inserts require affected-row evidence.
216#[allow(clippy::too_many_arguments)]
217pub(crate) fn vector_insert_statements_for_substrate(
218    table: &str,
219    namespace: &str,
220    subject_id: Uuid,
221    substrate: SubstrateKind,
222    field: &str,
223    embedding_model: &str,
224    embedding: &[f32],
225    label_prefix: &str,
226    insert_guard: Option<AffectedRowGuard>,
227) -> Vec<PlanStatement> {
228    let subject = subject_id.to_string();
229    let model_key = table
230        .strip_prefix("vec_")
231        .expect("runtime vector tables use the vec_ prefix");
232    let kind_str = substrate.to_string();
233    let blob = khive_storage::encode_f32_native(embedding);
234    vec![
235        // Delta-log any pre-existing row this REPLACE is about to evict —
236        // mirrors `log_vector_deletes`'s "replace" predicate exactly.
237        PlanStatement {
238            statement: SqlStatement {
239                sql: format!(
240                    "INSERT INTO ann_write_log \
241                     (namespace, embedding_model, kind, field, subject_id, op) \
242                     SELECT namespace, embedding_model, kind, field, subject_id, 'delete' \
243                     FROM {table} WHERE subject_id = ?1 AND NOT \
244                     (namespace = ?2 AND embedding_model = ?3 AND kind = ?4 AND field = ?5)"
245                ),
246                params: vec![
247                    SqlValue::Text(subject.clone()),
248                    SqlValue::Text(namespace.to_string()),
249                    SqlValue::Text(embedding_model.to_string()),
250                    SqlValue::Text(kind_str.clone()),
251                    SqlValue::Text(field.to_string()),
252                ],
253                label: Some(format!("{label_prefix}-log-delete")),
254            },
255            guard: None,
256        },
257        PlanStatement {
258            statement: SqlStatement {
259                sql: format!("DELETE FROM {table} WHERE subject_id = ?1"),
260                params: vec![SqlValue::Text(subject.clone())],
261                label: Some(format!("{label_prefix}-delete")),
262            },
263            guard: None,
264        },
265        // A replacement through this raw atomic plan does not know the source
266        // text, even if its new vector bytes happen to equal the old bytes.
267        PlanStatement {
268            statement: SqlStatement {
269                sql: "DELETE FROM vector_provenance \
270                      WHERE model_key = ?1 AND subject_id = ?2"
271                    .to_string(),
272                params: vec![
273                    SqlValue::Text(model_key.to_string()),
274                    SqlValue::Text(subject.clone()),
275                ],
276                label: Some(format!("{label_prefix}-clear-provenance")),
277            },
278            guard: None,
279        },
280        PlanStatement {
281            statement: SqlStatement {
282                sql: format!(
283                    "INSERT INTO {table} \
284                     (subject_id, namespace, kind, field, embedding_model, embedding) \
285                     VALUES (?1, ?2, ?3, ?4, ?5, ?6)"
286                ),
287                params: vec![
288                    SqlValue::Text(subject.clone()),
289                    SqlValue::Text(namespace.to_string()),
290                    SqlValue::Text(kind_str.clone()),
291                    SqlValue::Text(field.to_string()),
292                    SqlValue::Text(embedding_model.to_string()),
293                    SqlValue::Blob(blob),
294                ],
295                label: Some(format!("{label_prefix}-insert")),
296            },
297            guard: insert_guard,
298        },
299        PlanStatement {
300            statement: SqlStatement {
301                sql: "INSERT INTO ann_write_log \
302                      (namespace, embedding_model, kind, field, subject_id, op) \
303                      VALUES (?1, ?2, ?3, ?4, ?5, 'upsert')"
304                    .to_string(),
305                params: vec![
306                    SqlValue::Text(namespace.to_string()),
307                    SqlValue::Text(embedding_model.to_string()),
308                    SqlValue::Text(kind_str),
309                    SqlValue::Text(field.to_string()),
310                    SqlValue::Text(subject),
311                ],
312                label: Some(format!("{label_prefix}-log-upsert")),
313            },
314            guard: insert_guard,
315        },
316    ]
317}
318
319/// Build and commit `specs` as one atomic unit. Returns the persisted
320/// [`Note`]s in the same order as `specs`. On any failure — pre-write
321/// validation, embedding, or the atomic commit pass itself — NO note, FTS
322/// document, or vector row from any spec is left behind (embed failures
323/// occur before any write is attempted; commit-pass failures roll back the
324/// whole unit per [`crate::atomic_runner::run_atomic_unit`]'s guarantee).
325#[cfg(test)]
326pub(crate) async fn create_notes_atomic(
327    runtime: &KhiveRuntime,
328    specs: Vec<AtomicNoteSpec<'_>>,
329) -> RuntimeResult<Vec<Note>> {
330    Ok(create_notes_atomic_with_report(runtime, specs).await?.0)
331}
332
333/// Build and commit notes with an embedding-truncation report. The report keeps
334/// logical per-note/model accounting even when identical content shares one
335/// provider result, and is returned only when the whole note set commits
336/// successfully.
337pub async fn create_notes_atomic_with_report(
338    runtime: &KhiveRuntime,
339    specs: Vec<AtomicNoteSpec<'_>>,
340) -> RuntimeResult<(Vec<Note>, crate::retrieval::EmbeddingTruncationReport)> {
341    create_notes_atomic_with_attachments(runtime, specs, &[]).await
342}
343
344/// Create notes and their role-keyed attachments in the same writer transaction.
345/// The existing note, FTS, vector and revision accounting is preserved.
346pub async fn create_notes_atomic_with_attachments(
347    runtime: &KhiveRuntime,
348    specs: Vec<AtomicNoteSpec<'_>>,
349    attachments: &[khive_storage::NewAttachment],
350) -> RuntimeResult<(Vec<Note>, crate::retrieval::EmbeddingTruncationReport)> {
351    create_notes_atomic_with_extra_statements(runtime, specs, attachments, Vec::new()).await
352}
353
354/// Like [`create_notes_atomic_with_attachments`], and also commit `extra_statements` in the same
355/// writer transaction, after the last note's plan. Each statement must affect exactly one row;
356/// if any statement fails, no note and no extra row is left behind.
357pub async fn create_notes_atomic_with_extra_statements(
358    runtime: &KhiveRuntime,
359    specs: Vec<AtomicNoteSpec<'_>>,
360    attachments: &[khive_storage::NewAttachment],
361    extra_statements: Vec<khive_storage::SqlStatement>,
362) -> RuntimeResult<(Vec<Note>, crate::retrieval::EmbeddingTruncationReport)> {
363    validate_note_attachments(runtime, attachments)?;
364    let mut prepared = prepare_atomic_notes(runtime, specs, AtomicNoteOptions::default()).await?;
365    append_note_attachments(&mut prepared, attachments)?;
366    append_extra_statements(&mut prepared, extra_statements)?;
367    match crate::atomic_runner::run_atomic_unit_with_note_versions(
368        runtime.sql().as_ref(),
369        prepared.plans,
370        true,
371    )
372    .await
373    {
374        Ok((AtomicRunOutcome::Committed { .. }, versions)) => {
375            assert_eq!(
376                prepared.notes.len(),
377                versions.len(),
378                "one revision receipt per prepared note"
379            );
380            for (note, version) in prepared.notes.iter_mut().zip(versions) {
381                note.version = version;
382            }
383            Ok((prepared.notes, prepared.embedding_truncation))
384        }
385        Ok((
386            AtomicRunOutcome::RolledBack {
387                failed_op_index,
388                failure,
389            },
390            _,
391        )) => Err(RuntimeError::Internal(format!(
392            "atomic multi-note write rolled back at op {failed_op_index}: {failure:?}"
393        ))),
394        Err(e) => Err(RuntimeError::Storage(e.0)),
395    }
396}
397
398pub(crate) fn validate_note_attachments(
399    runtime: &KhiveRuntime,
400    attachments: &[khive_storage::NewAttachment],
401) -> RuntimeResult<()> {
402    if !attachments.is_empty() {
403        // Note plans execute on runtime.sql(); attachment liveness belongs to
404        // main. Refuse a split placement before preparing any write.
405        runtime.attachments()?;
406    }
407    for attachment in attachments {
408        attachment.validate()?;
409    }
410    Ok(())
411}
412
413pub(crate) fn append_note_attachments(
414    prepared: &mut PreparedAtomicNotes,
415    attachments: &[khive_storage::NewAttachment],
416) -> RuntimeResult<()> {
417    for (plan, note) in prepared.plans.iter_mut().zip(&prepared.notes) {
418        let AtomicOpPlan::AddNote(plan) = plan else {
419            return Err(RuntimeError::Internal(
420                "expected prepared attachment owner note".into(),
421            ));
422        };
423        for attachment in attachments {
424            let row = khive_storage::Attachment::from_new(
425                note.id,
426                khive_storage::AttachmentSubstrate::Note,
427                attachment.clone(),
428                note.created_at,
429            );
430            // Append after the note's complete index plan, preserving adjacent
431            // last_insert_rowid-dependent FTS statements and their counters.
432            plan.statements.push(PlanStatement {
433                statement: khive_db::stores::attachment::attachment_upsert_statement(&row)?,
434                guard: Some(AffectedRowGuard::exactly(1)),
435            });
436        }
437    }
438    Ok(())
439}
440
441/// Append caller-built statements to the last note's plan so they commit in the same unit. Each
442/// statement must affect exactly one row.
443pub(crate) fn append_extra_statements(
444    prepared: &mut PreparedAtomicNotes,
445    statements: Vec<khive_storage::SqlStatement>,
446) -> RuntimeResult<()> {
447    if statements.is_empty() {
448        return Ok(());
449    }
450    let Some(AtomicOpPlan::AddNote(last)) = prepared.plans.last_mut() else {
451        return Err(RuntimeError::Internal(
452            "expected prepared note for extra statements".into(),
453        ));
454    };
455    for statement in statements {
456        last.statements.push(PlanStatement {
457            statement,
458            guard: Some(AffectedRowGuard::exactly(1)),
459        });
460    }
461    Ok(())
462}
463
464pub(crate) async fn prepare_atomic_notes(
465    runtime: &KhiveRuntime,
466    specs: Vec<AtomicNoteSpec<'_>>,
467    options: AtomicNoteOptions<'_>,
468) -> RuntimeResult<PreparedAtomicNotes> {
469    validate_atomic_note_options(&options)?;
470    if options.embed != Some(false) {
471        if let Some(model) = options.embedding_model {
472            runtime.resolve_embedding_model(Some(model))?;
473        }
474    }
475    prepare_atomic_note_requests(
476        runtime,
477        specs
478            .into_iter()
479            .map(|spec| AtomicNoteRequest { spec, options })
480            .collect(),
481    )
482    .await
483}
484
485fn validate_atomic_note_options(options: &AtomicNoteOptions<'_>) -> RuntimeResult<()> {
486    if let Some(value) = options.salience {
487        if !value.is_finite() || !(0.0..=1.0).contains(&value) {
488            return Err(RuntimeError::InvalidInput(
489                "salience must be a finite value in [0.0, 1.0]".into(),
490            ));
491        }
492    }
493    if let Some(value) = options.decay_factor {
494        if !value.is_finite() || value < 0.0 {
495            return Err(RuntimeError::InvalidInput(
496                "decay_factor must be a finite value >= 0.0".into(),
497            ));
498        }
499    }
500    Ok(())
501}
502
503pub(crate) async fn prepare_atomic_note_requests(
504    runtime: &KhiveRuntime,
505    requests: Vec<AtomicNoteRequest<'_>>,
506) -> RuntimeResult<PreparedAtomicNotes> {
507    let mut embed_model_names = Vec::new();
508    let mut note_models = Vec::with_capacity(requests.len());
509    for request in &requests {
510        let options = &request.options;
511        validate_atomic_note_options(options)?;
512        let models = if options.embed == Some(false) {
513            Vec::new()
514        } else if let Some(model) = options.embedding_model {
515            runtime.resolve_embedding_model(Some(model))?;
516            vec![model.to_owned()]
517        } else {
518            runtime.embedding_models_for_note_kind(request.spec.kind)
519        };
520        let mut indices = Vec::with_capacity(models.len());
521        for model in models {
522            let index = match embed_model_names.iter().position(|name| name == &model) {
523                Some(index) => index,
524                None => {
525                    embed_model_names.push(model);
526                    embed_model_names.len() - 1
527                }
528            };
529            indices.push(index);
530        }
531        note_models.push(indices);
532    }
533    // ---- 1. Validate + build Note objects (all pre-write checks, same as
534    // create_note_inner, before any embedding or DML is attempted). ----
535    let mut notes: Vec<Note> = Vec::with_capacity(requests.len());
536    for (index, request) in requests.iter().enumerate() {
537        let spec = &request.spec;
538        let options = &request.options;
539        let record = format!("note[{index}]");
540        runtime.validate_note_kind(spec.kind)?;
541        // Same owned-identity derivation every other note-write site runs
542        // (`operations.rs`'s create funnel, `atomic_prepare::prepare_add_note`):
543        // derive from the token BEFORE the secret scan and note construction,
544        // so a caller-supplied `from_actor`/`thread_id`/etc. in `spec.properties`
545        // never reaches storage verbatim on this writer either. Skipped when
546        // the caller already derived (`properties_already_derived`): the
547        // validator is not guaranteed idempotent, so a second call for the
548        // same request is a different candidate, not a no-op.
549        let properties = if options.properties_already_derived {
550            spec.properties.clone()
551        } else {
552            runtime.derive_note_write_properties(spec.kind, spec.token, spec.properties.clone())?
553        };
554        crate::secret_gate::reject_reserved_secret_gate_property(properties.as_ref())?;
555        crate::secret_gate::check_note_fields_at(
556            spec.content,
557            spec.name,
558            properties.as_ref(),
559            &record,
560        )?;
561
562        let ns = spec.token.namespace().as_str();
563        let mut note = Note::new(ns, spec.kind, spec.content);
564        note.key = options.key.map(str::to_owned);
565        if let Some(salience) = options.salience {
566            note = note.with_salience(salience);
567        }
568        if let Some(decay_factor) = options.decay_factor {
569            note = note.with_decay(decay_factor);
570        }
571        if let Some(id) = spec.id {
572            note.id = id;
573        }
574        if let Some(n) = spec.name {
575            note = note.with_name(n);
576        }
577        if let Some(p) = properties {
578            note = note.with_properties(p);
579        }
580        notes.push(note);
581    }
582
583    // Deferred key publication clears the temporary Note.key; its private receipt
584    // option is the original keyed-memory intent. Generic and stream routes
585    // retain options.key. Kind is always the canonical prepared note kind.
586    let memory_receipts: Vec<bool> = notes
587        .iter()
588        .zip(&requests)
589        .map(|(note, request)| {
590            note.kind == "memory"
591                && (request.options.key.is_some() || request.options.memory_visibility_receipt)
592        })
593        .collect();
594    if memory_receipts.iter().any(|capture| *capture) {
595        runtime.require_visibility_cutover()?;
596    }
597
598    // ---- 2. Batch distinct content per model in parallel, BEFORE
599    // opening any transaction. Any failure aborts here — no write has been
600    // attempted. Identical note siblings reuse the same computed vector. ----
601    let mut content_group_by_text: HashMap<&str, usize> = HashMap::new();
602    let mut content_groups: Vec<Vec<usize>> = Vec::new();
603    let mut note_content_groups: Vec<usize> = Vec::with_capacity(notes.len());
604    for (note_idx, note) in notes.iter().enumerate() {
605        let text = requests[note_idx]
606            .options
607            .embedding_content
608            .unwrap_or_else(|| crate::curation::note_embedding_text_ref(note));
609        let content_group_idx = match content_group_by_text.get(text) {
610            Some(&idx) => {
611                content_groups[idx].push(note_idx);
612                idx
613            }
614            None => {
615                let idx = content_groups.len();
616                content_group_by_text.insert(text, idx);
617                content_groups.push(vec![note_idx]);
618                idx
619            }
620        };
621        note_content_groups.push(content_group_idx);
622    }
623
624    // (content_group_idx, model_idx) -> outcome, filled as tasks complete.
625    let mut embedding_outcomes: Vec<Vec<Option<crate::retrieval::DocumentEmbeddingOutcome>>> =
626        content_groups
627            .iter()
628            .map(|_| vec![None; embed_model_names.len()])
629            .collect();
630    let mut embedding_truncation = crate::retrieval::EmbeddingTruncationReport::default();
631
632    // `requests` is empty when a caller prepares a note set every member of
633    // which was refused before preparation: there is nothing to embed, no
634    // token to read, and the lazy vector-table create belongs to a write
635    // that is not going to happen.
636    if !embed_model_names.is_empty() {
637        // Ensure every model's vector table exists before the commit pass —
638        // the same lazy-create side effect as vector-store construction, with
639        // all model tables sharing one schema-writer acquisition.
640        let models = embed_model_names
641            .iter()
642            .map(|name| {
643                runtime
644                    .vector_model_metadata(name)
645                    .map(|(name, dimensions)| (crate::config::sanitize_key(&name), dimensions))
646            })
647            .collect::<RuntimeResult<Vec<_>>>()?;
648        let model_specs: Vec<_> = models
649            .iter()
650            .map(|(key, dimensions)| (key.as_str(), *dimensions))
651            .collect();
652        runtime.backend().ensure_vector_tables(&model_specs)?;
653
654        let usage_ctx = crate::usage::current();
655        let mut join_set = tokio::task::JoinSet::new();
656        for (model_idx, model_name) in embed_model_names.iter().enumerate() {
657            let mut groups = Vec::new();
658            let mut texts = Vec::new();
659            let mut token = None;
660            for (group_idx, note_indices) in content_groups.iter().enumerate() {
661                let Some(&note_idx) = note_indices
662                    .iter()
663                    .find(|&&index| note_models[index].contains(&model_idx))
664                else {
665                    continue;
666                };
667                token.get_or_insert_with(|| requests[note_idx].spec.token.clone());
668                groups.push(group_idx);
669                texts.push(
670                    requests[note_idx]
671                        .options
672                        .embedding_content
673                        .unwrap_or_else(|| {
674                            crate::curation::note_embedding_text_ref(&notes[note_idx])
675                        })
676                        .to_owned(),
677                );
678            }
679            let rt = runtime.clone();
680            let token = token.expect("a selected model has at least one note");
681            let name = model_name.clone();
682            let ctx = usage_ctx.clone();
683            join_set.spawn(crate::runtime::inherit_request_embedder_scope(async move {
684                let fut = async {
685                    let mut outcomes = Vec::with_capacity(texts.len());
686                    for chunk in texts.chunks(lattice_embed::DEFAULT_MAX_BATCH_SIZE) {
687                        outcomes.extend(
688                            rt.embed_document_batch_with_model_outcomes_for_token(
689                                &token, &name, chunk,
690                            )
691                            .await?,
692                        );
693                    }
694                    Ok::<_, RuntimeError>(outcomes)
695                };
696                let result = match ctx {
697                    Some(ctx) => crate::usage::scope(ctx, fut).await,
698                    None => fut.await,
699                };
700                (groups, model_idx, result)
701            }));
702        }
703
704        while let Some(joined) = join_set.join_next().await {
705            match joined {
706                Ok((groups, model_idx, Ok(outcomes))) => {
707                    for (group_idx, outcome) in groups.into_iter().zip(outcomes) {
708                        embedding_outcomes[group_idx][model_idx] = Some(outcome);
709                    }
710                }
711                Ok((_, _, Err(e))) => {
712                    join_set.abort_all();
713                    return Err(e);
714                }
715                Err(join_err) => {
716                    join_set.abort_all();
717                    return Err(RuntimeError::Internal(format!(
718                        "embed task panicked: {join_err}"
719                    )));
720                }
721            }
722        }
723    }
724
725    // Preserve the report's logical note/model accounting even when identical
726    // siblings share one provider invocation.
727    for (note_idx, &content_group_idx) in note_content_groups.iter().enumerate() {
728        for &model_idx in &note_models[note_idx] {
729            let outcome = embedding_outcomes[content_group_idx][model_idx]
730                .as_ref()
731                .expect("every requested embedding was observed");
732            embedding_truncation.observe(outcome);
733            if outcome.truncated {
734                tracing::warn!(
735                    model = %outcome.model_name,
736                    source_bytes = outcome.source_bytes,
737                    embedded_bytes = outcome.embedded_bytes,
738                    "atomic note embedding input truncated; full content will be stored unchanged"
739                );
740            }
741        }
742    }
743
744    // Reject any non-finite embedding value BEFORE any plan is built — same
745    // validation the canonical `VectorStore::insert` DML performs, applied
746    // here so a bad custom embedding provider never reaches the raw vector
747    // insert on this path (embed-first: no note, FTS document, or vector row
748    // has been written yet).
749    for outcomes_for_content in &embedding_outcomes {
750        for outcome in outcomes_for_content.iter().flatten() {
751            if let Some(idx) = non_finite_index(&outcome.vector) {
752                return Err(non_finite_vector_error(idx, outcome.vector[idx]));
753            }
754        }
755    }
756
757    // ---- 3. Build one AddNotePlan per spec (row + FTS + vector-insert
758    // statements), all pre-computed embeddings already in hand. ----
759    let mut plans: Vec<AtomicOpPlan> = Vec::with_capacity(notes.len());
760    for (note_idx, note) in notes.iter().enumerate() {
761        let outcomes_for_note = &embedding_outcomes[note_content_groups[note_idx]];
762        let mut statements = vec![PlanStatement {
763            statement: if note.key.is_some() {
764                khive_db::stores::note::note_insert_keyed_statement(note)
765            } else {
766                khive_db::stores::note::note_upsert_statement(note)
767            },
768            guard: Some(AffectedRowGuard::exactly(1)),
769        }];
770
771        if memory_receipts[note_idx] {
772            statements.push(PlanStatement {
773                statement: SqlStatement {
774                    sql:
775                        "INSERT INTO memory_visibility_receipts (namespace, note_id, model_count) \
776                          VALUES (?1, ?2, ?3)"
777                            .into(),
778                    params: vec![
779                        SqlValue::Text(note.namespace.clone()),
780                        SqlValue::Text(note.id.to_string()),
781                        SqlValue::Integer(note_models[note_idx].len() as i64),
782                    ],
783                    label: Some("memory-visibility-receipt".into()),
784                },
785                guard: Some(AffectedRowGuard::exactly(1)),
786            });
787        }
788
789        if let Some(fault) = maybe_inject_fts_failure(&note.namespace, "fault-injected-fts") {
790            statements.push(fault);
791        } else {
792            // Delete-then-insert upsert — mirrors `text.rs`'s
793            // `upsert_document_dml` exactly, so a caller-supplied/reused note
794            // id never leaves a stale/duplicate FTS document behind (the
795            // note row itself is already an upsert via
796            // `note_upsert_statement`). The delete alone is enough on the
797            // sidecar rowid-map side too — the insert pair's `INSERT OR
798            // REPLACE` below overwrites the map row rather than requiring a
799            // prior delete of it (see `delete_document_statement`'s doc
800            // comment).
801            statements.push(PlanStatement {
802                statement: khive_db::stores::text::delete_document_statement(
803                    "fts_notes",
804                    &note.namespace,
805                    note.id,
806                ),
807                guard: None,
808            });
809            // Order-sensitive pair: the map upsert's `last_insert_rowid()`
810            // must read back the FTS insert immediately before it, with
811            // nothing else written to this connection in between — pushed
812            // here as the array `insert_document_statements` returns so the
813            // two can never be separated by an edit to this call site.
814            for statement in khive_db::stores::text::insert_document_statements(
815                "fts_notes",
816                &note_fts_document(note),
817            ) {
818                statements.push(PlanStatement {
819                    statement,
820                    guard: None,
821                });
822            }
823        }
824
825        if let Some(fault) = maybe_inject_vector_failure(&note.namespace, "fault-injected-vector") {
826            statements.push(fault);
827        } else {
828            for &model_idx in &note_models[note_idx] {
829                let model_name = &embed_model_names[model_idx];
830                let outcome = outcomes_for_note[model_idx]
831                    .as_ref()
832                    .expect("every model index observed exactly once");
833                let table = format!("vec_{}", crate::config::sanitize_key(model_name));
834                statements.extend(vector_insert_statements(
835                    &table,
836                    &note.namespace,
837                    note.id,
838                    "note.content",
839                    model_name,
840                    &outcome.vector,
841                    &format!("atomic-message-vec-{table}-{}", note.id),
842                ));
843                if memory_receipts[note_idx] {
844                    // The preceding statement is the model's ann_write_log
845                    // upsert. Keep this insert adjacent: last_insert_rowid()
846                    // reads that exact transaction-local sequence, never a
847                    // later MAX(seq) vulnerable to concurrency/compaction.
848                    statements.push(PlanStatement {
849                        statement: SqlStatement {
850                            sql: "INSERT INTO memory_visibility_fences \
851                                  (namespace, note_id, model, ann_write_log_seq) \
852                                  VALUES (?1, ?2, ?3, last_insert_rowid())"
853                                .into(),
854                            params: vec![
855                                SqlValue::Text(note.namespace.clone()),
856                                SqlValue::Text(note.id.to_string()),
857                                SqlValue::Text(model_name.clone()),
858                            ],
859                            label: Some("memory-visibility-fence".into()),
860                        },
861                        guard: Some(AffectedRowGuard::exactly(1)),
862                    });
863                }
864            }
865        }
866
867        // After every log/fence pair: inserting the independent marker must
868        // never disturb either ANN or FTS last_insert_rowid adjacency.
869        if memory_receipts[note_idx] {
870            statements.push(PlanStatement {
871                statement: SqlStatement {
872                    sql: "INSERT INTO memory_visibility_epochs (note_id, namespace, epoch) \
873                          VALUES (?1, ?2, 'modern')"
874                        .into(),
875                    params: vec![
876                        SqlValue::Text(note.id.to_string()),
877                        SqlValue::Text(note.namespace.clone()),
878                    ],
879                    label: Some("memory-visibility-epoch".into()),
880                },
881                guard: Some(AffectedRowGuard::exactly(1)),
882            });
883        }
884
885        plans.push(AtomicOpPlan::AddNote(Box::new(AddNotePlan {
886            note_guard: Some(crate::note_write::NoteWriteGuard {
887                namespace: requests[note_idx].spec.token.namespace().as_str().into(),
888                target_id: note.id,
889                expected_version: None,
890                fence: requests[note_idx].options.fence.cloned(),
891                create_key: note
892                    .key
893                    .as_ref()
894                    .map(|key| crate::note_write::CreateKeyClaim {
895                        kind: note.kind.clone(),
896                        key: key.clone(),
897                        content: note.content.clone(),
898                        properties: note.properties.clone(),
899                        replay_signal: requests[note_idx].options.replay_receipt,
900                    }),
901            }),
902            note_id: note.id,
903            statements,
904            post_commit: PostCommitEffect::None,
905        })));
906    }
907
908    Ok(PreparedAtomicNotes {
909        notes,
910        plans,
911        embedding_truncation,
912    })
913}
914
915#[cfg(test)]
916#[path = "atomic_message_tests.rs"]
917mod tests;