Skip to main content

khive_runtime/
atomic_message.rs

1//! Atomic multi-note write primitive: commits a set of notes — each with its
2//! FTS document and every registered embedding model's vector row — in ONE
3//! writer transaction, instead of one `create_note` call per note.
4//!
5//! Built for `khive-pack-comm`'s `dual_write_message` (outbound + inbound
6//! copy of a `comm.send`/`comm.reply`), which previously cost roughly a
7//! dozen separate writer acquisitions per send: two `create_note_inner`
8//! calls (row + FTS + one vector insert per registered model each) plus a
9//! root-send `thread_id` patch. Shaped as `create_notes_atomic(Vec<AtomicNoteSpec>)`
10//! rather than a comm-specific pair primitive so other multi-write verbs
11//! (`memory.remember`, `gtd.assign`) can adopt it later without a new type.
12//!
13//! # Embed-first
14//!
15//! Embedding is slow compute (network/model calls). Every distinct content's
16//! embeddings, across every registered model, are computed **before** any
17//! transaction opens — the writer is held only for synchronous DML, exactly
18//! like the rest of the ADR-099 atomic-unit machinery
19//! (`atomic_plan`/`atomic_runner`).
20//! This is the same reason `atomic_prepare::prepare_add_note` defers vector
21//! indexing to a post-commit `PostCommitEffect::ReindexNote`; the difference
22//! here is embeddings are computed *before* commit instead of *after*, so
23//! the vector rows land in the SAME atomic unit as the note row and FTS
24//! document, and no post-commit reindex is needed at all — every plan built
25//! by [`create_notes_atomic`] carries `post_commit: PostCommitEffect::None`.
26//!
27//! # One writer acquisition
28//!
29//! Each note becomes its own [`AddNotePlan`]; every spec's plan is applied by
30//! ONE [`run_atomic_unit`] call — one [`khive_storage::SqlAccess::atomic_unit`],
31//! one writer checkout, one WAL commit for the whole set. A failure on any
32//! note's plan rolls back the ENTIRE unit (`atomic_runner`'s documented
33//! guarantee: a later op's failure unwinds even an earlier op's own
34//! already-`RELEASE`d `SAVEPOINT`), so a crash or guard failure partway
35//! through never leaves an orphan copy — the process-crash gap the two-call
36//! version of `dual_write_message` used to document is closed by
37//! construction.
38
39use std::{collections::HashMap, sync::Arc};
40
41use serde_json::Value;
42use uuid::Uuid;
43
44use khive_storage::note::Note;
45use khive_storage::types::SqlValue;
46use khive_storage::{SqlStatement, StorageCapability, StorageError};
47use khive_types::SubstrateKind;
48
49use crate::atomic_plan::{AddNotePlan, AffectedRowGuard, PlanStatement, PostCommitEffect};
50use crate::atomic_runner::{run_atomic_unit, AtomicOpPlan, AtomicRunOutcome};
51use crate::config::NamespaceToken;
52use crate::curation::note_fts_document;
53use crate::error::{RuntimeError, RuntimeResult};
54use crate::runtime::KhiveRuntime;
55
56/// One note to write as part of an atomic set. Mirrors the subset of
57/// `create_note_inner`'s parameters `comm.send`/`comm.reply` actually use —
58/// no `annotates`, `salience`, `decay_factor`, `embedding_content` override,
59/// or explicit `embedding_model` pin. A future caller needing those can grow
60/// this struct; none of khive-pack-comm's call sites need them today.
61pub struct AtomicNoteSpec<'a> {
62    /// Namespace + actor identity this note is written under.
63    pub token: &'a NamespaceToken,
64    /// Caller-supplied id, when the id must be known before the write (e.g.
65    /// a root send's own id becomes the canonical `thread_id` stored in
66    /// BOTH copies' properties). `None` generates a fresh id, same as
67    /// `Note::new`.
68    pub id: Option<Uuid>,
69    pub kind: &'a str,
70    pub name: Option<&'a str>,
71    pub content: &'a str,
72    pub properties: Option<Value>,
73}
74
75/// Bit-identical to `khive-db`'s private `f32_slice_as_bytes` (native-endian
76/// reinterpretation of the float buffer) but built from safe `to_ne_bytes`
77/// calls instead of an unsafe pointer cast — this crate has no visibility
78/// into that helper, and correctness here only requires this process's
79/// writes and reads agree on layout, which native-endian byte-for-byte
80/// concatenation guarantees identically to the raw cast.
81fn f32_vec_to_bytes(data: &[f32]) -> Vec<u8> {
82    let mut bytes = Vec::with_capacity(data.len() * 4);
83    for f in data {
84        bytes.extend_from_slice(&f.to_ne_bytes());
85    }
86    bytes
87}
88
89/// Mirrors `khive-db`'s private `non_finite_index` — this crate has no
90/// visibility into that helper, so the check is reproduced here to keep the
91/// atomic path's embedding validation observably identical to the canonical
92/// `create_note_inner` -> `VectorStore::insert` path.
93fn non_finite_index(data: &[f32]) -> Option<usize> {
94    data.iter().position(|v| !v.is_finite())
95}
96
97/// Same error shape (capability, operation label, message) as `khive-db`'s
98/// private `non_finite_vector_error("vec_insert", ..)` — the atomic path
99/// must reject a non-finite embedding exactly as the raw `VectorStore::insert`
100/// DML does, not silently write it.
101fn non_finite_vector_error(idx: usize, value: f32) -> RuntimeError {
102    RuntimeError::Storage(StorageError::InvalidInput {
103        capability: StorageCapability::Vectors,
104        operation: "vec_insert".into(),
105        message: format!(
106            "non-finite value at index {idx}: {value} \
107             (NaN/Inf values corrupt distance computations)"
108        ),
109    })
110}
111
112/// A harmless no-op `UPDATE` with a statically zero-row predicate (`WHERE 1
113/// = 0`, not a match against a specific id), guarded `exactly(1)` so it
114/// always fails its guard inside the atomic unit regardless of which note
115/// ids exist in the store — a fixed nil-UUID predicate would pass its guard
116/// if a caller ever supplied that id. Spliced in by test-only fault
117/// injection (see [`maybe_inject_fts_failure`]/[`maybe_inject_vector_failure`])
118/// to prove the whole unit — not just the failing note's own plan — rolls
119/// back.
120///
121/// `fault-injection` is an ordinary Cargo feature (see this crate's
122/// `Cargo.toml`); it must never be enabled in a packaged/release build —
123/// doing so would compile this fault-injection path into a shipped binary.
124#[cfg(any(test, feature = "fault-injection"))]
125fn injected_failure_statement(label: &str) -> PlanStatement {
126    PlanStatement {
127        statement: SqlStatement {
128            sql: "UPDATE notes SET updated_at = updated_at WHERE 1 = 0".to_string(),
129            params: vec![],
130            label: Some(label.to_string()),
131        },
132        guard: Some(AffectedRowGuard::exactly(1)),
133    }
134}
135
136#[cfg(any(test, feature = "fault-injection"))]
137fn maybe_inject_fts_failure(namespace: &str, label: &str) -> Option<PlanStatement> {
138    crate::operations::consume_fts_fail_fault(namespace).then(|| injected_failure_statement(label))
139}
140#[cfg(not(any(test, feature = "fault-injection")))]
141fn maybe_inject_fts_failure(_namespace: &str, _label: &str) -> Option<PlanStatement> {
142    None
143}
144
145#[cfg(any(test, feature = "fault-injection"))]
146fn maybe_inject_vector_failure(namespace: &str, label: &str) -> Option<PlanStatement> {
147    crate::operations::consume_vector_fail_fault(namespace)
148        .then(|| injected_failure_statement(label))
149}
150#[cfg(not(any(test, feature = "fault-injection")))]
151fn maybe_inject_vector_failure(_namespace: &str, _label: &str) -> Option<PlanStatement> {
152    None
153}
154
155/// DELETE-then-INSERT-then-log statements for one (note, model) vector row.
156/// This static atomic plan cannot branch on an identity-constrained DELETE's
157/// affected-row count, so it retains the general delete-log scan used before
158/// `khive-db::stores::vectors::replace_vector_row_dml` gained its live-
159/// connection fast path. It uses the same plain-[`PlanStatement`] technique
160/// as `atomic_prepare::push_index_purge_statements`. Unguarded: same
161/// convention as the FTS-insert statement in
162/// `atomic_prepare::prepare_add_note` (the row's existence guard is carried by
163/// the plan's primary note-row statement, applied first).
164#[allow(clippy::too_many_arguments)]
165fn vector_insert_statements(
166    table: &str,
167    namespace: &str,
168    subject_id: Uuid,
169    field: &str,
170    embedding_model: &str,
171    embedding: &[f32],
172    label_prefix: &str,
173) -> Vec<PlanStatement> {
174    let subject = subject_id.to_string();
175    let kind_str = SubstrateKind::Note.to_string();
176    let blob = f32_vec_to_bytes(embedding);
177    vec![
178        // Delta-log any pre-existing row this REPLACE is about to evict —
179        // mirrors `log_vector_deletes`'s "replace" predicate exactly.
180        PlanStatement {
181            statement: SqlStatement {
182                sql: format!(
183                    "INSERT INTO ann_write_log \
184                     (namespace, embedding_model, kind, field, subject_id, op) \
185                     SELECT namespace, embedding_model, kind, field, subject_id, 'delete' \
186                     FROM {table} WHERE subject_id = ?1 AND NOT \
187                     (namespace = ?2 AND embedding_model = ?3 AND kind = ?4 AND field = ?5)"
188                ),
189                params: vec![
190                    SqlValue::Text(subject.clone()),
191                    SqlValue::Text(namespace.to_string()),
192                    SqlValue::Text(embedding_model.to_string()),
193                    SqlValue::Text(kind_str.clone()),
194                    SqlValue::Text(field.to_string()),
195                ],
196                label: Some(format!("{label_prefix}-log-delete")),
197            },
198            guard: None,
199        },
200        PlanStatement {
201            statement: SqlStatement {
202                sql: format!("DELETE FROM {table} WHERE subject_id = ?1"),
203                params: vec![SqlValue::Text(subject.clone())],
204                label: Some(format!("{label_prefix}-delete")),
205            },
206            guard: None,
207        },
208        PlanStatement {
209            statement: SqlStatement {
210                sql: format!(
211                    "INSERT INTO {table} \
212                     (subject_id, namespace, kind, field, embedding_model, embedding) \
213                     VALUES (?1, ?2, ?3, ?4, ?5, ?6)"
214                ),
215                params: vec![
216                    SqlValue::Text(subject.clone()),
217                    SqlValue::Text(namespace.to_string()),
218                    SqlValue::Text(kind_str.clone()),
219                    SqlValue::Text(field.to_string()),
220                    SqlValue::Text(embedding_model.to_string()),
221                    SqlValue::Blob(blob),
222                ],
223                label: Some(format!("{label_prefix}-insert")),
224            },
225            guard: None,
226        },
227        PlanStatement {
228            statement: SqlStatement {
229                sql: "INSERT INTO ann_write_log \
230                      (namespace, embedding_model, kind, field, subject_id, op) \
231                      VALUES (?1, ?2, ?3, ?4, ?5, 'upsert')"
232                    .to_string(),
233                params: vec![
234                    SqlValue::Text(namespace.to_string()),
235                    SqlValue::Text(embedding_model.to_string()),
236                    SqlValue::Text(kind_str),
237                    SqlValue::Text(field.to_string()),
238                    SqlValue::Text(subject),
239                ],
240                label: Some(format!("{label_prefix}-log-upsert")),
241            },
242            guard: None,
243        },
244    ]
245}
246
247/// Build and commit `specs` as one atomic unit. Returns the persisted
248/// [`Note`]s in the same order as `specs`. On any failure — pre-write
249/// validation, embedding, or the atomic commit pass itself — NO note, FTS
250/// document, or vector row from any spec is left behind (embed failures
251/// occur before any write is attempted; commit-pass failures roll back the
252/// whole unit per [`run_atomic_unit`]'s guarantee).
253pub async fn create_notes_atomic(
254    runtime: &KhiveRuntime,
255    specs: Vec<AtomicNoteSpec<'_>>,
256) -> RuntimeResult<Vec<Note>> {
257    Ok(create_notes_atomic_with_report(runtime, specs).await?.0)
258}
259
260/// The truncation-reporting form of [`create_notes_atomic`]. The report keeps
261/// logical per-note/model accounting even when identical content shares one
262/// provider result, and is returned only when the whole note set commits
263/// successfully.
264pub async fn create_notes_atomic_with_report(
265    runtime: &KhiveRuntime,
266    specs: Vec<AtomicNoteSpec<'_>>,
267) -> RuntimeResult<(Vec<Note>, crate::retrieval::EmbeddingTruncationReport)> {
268    // ---- 1. Validate + build Note objects (all pre-write checks, same as
269    // create_note_inner, before any embedding or DML is attempted). ----
270    let mut notes: Vec<Note> = Vec::with_capacity(specs.len());
271    for spec in &specs {
272        runtime.validate_note_kind(spec.kind)?;
273        // Same owned-identity derivation every other note-write site runs
274        // (`operations.rs`'s create funnel, `atomic_prepare::prepare_add_note`):
275        // derive from the token BEFORE the secret scan and note construction,
276        // so a caller-supplied `from_actor`/`thread_id`/etc. in `spec.properties`
277        // never reaches storage verbatim on this writer either.
278        let properties =
279            runtime.derive_note_write_properties(spec.kind, spec.token, spec.properties.clone())?;
280        crate::secret_gate::reject_reserved_secret_gate_property(properties.as_ref())?;
281        crate::secret_gate::check(spec.content)?;
282        if let Some(n) = spec.name {
283            crate::secret_gate::check(n)?;
284        }
285        if let Some(ref p) = properties {
286            crate::secret_gate::check_json(p)?;
287        }
288
289        let ns = spec.token.namespace().as_str();
290        let mut note = Note::new(ns, spec.kind, spec.content);
291        if let Some(id) = spec.id {
292            note.id = id;
293        }
294        if let Some(n) = spec.name {
295            note = note.with_name(n);
296        }
297        if let Some(p) = properties {
298            note = note.with_properties(p);
299        }
300        notes.push(note);
301    }
302
303    // ---- 2. Embed every distinct (content, model) pair in parallel, BEFORE
304    // opening any transaction. Any failure aborts here — no write has been
305    // attempted. Identical note siblings reuse the same computed vector. ----
306    let embed_model_names = runtime.registered_embedding_model_names();
307    let mut content_group_by_text: HashMap<&str, usize> = HashMap::new();
308    let mut content_groups: Vec<Vec<usize>> = Vec::new();
309    let mut note_content_groups: Vec<usize> = Vec::with_capacity(notes.len());
310    for (note_idx, note) in notes.iter().enumerate() {
311        let text = crate::curation::note_embedding_text_ref(note);
312        let content_group_idx = match content_group_by_text.get(text) {
313            Some(&idx) => {
314                content_groups[idx].push(note_idx);
315                idx
316            }
317            None => {
318                let idx = content_groups.len();
319                content_group_by_text.insert(text, idx);
320                content_groups.push(vec![note_idx]);
321                idx
322            }
323        };
324        note_content_groups.push(content_group_idx);
325    }
326
327    // (content_group_idx, model_idx) -> outcome, filled as tasks complete.
328    let mut embedding_outcomes: Vec<Vec<Option<crate::retrieval::DocumentEmbeddingOutcome>>> =
329        content_groups
330            .iter()
331            .map(|_| vec![None; embed_model_names.len()])
332            .collect();
333    let mut embedding_truncation = crate::retrieval::EmbeddingTruncationReport::default();
334
335    if !embed_model_names.is_empty() {
336        // Ensure every model's vector table exists before the commit pass —
337        // the same lazy-create side effect `vectors_for_model` performs on
338        // the non-atomic path, done once per model rather than per note.
339        for model_name in &embed_model_names {
340            runtime.vectors_for_model(specs[0].token, model_name)?;
341        }
342
343        let usage_ctx = crate::usage::current();
344        let mut join_set = tokio::task::JoinSet::new();
345        for (content_group_idx, note_indices) in content_groups.iter().enumerate() {
346            let note_idx = note_indices[0];
347            let spec = &specs[note_idx];
348            let note = &notes[note_idx];
349            // Spawned tasks need owned text; share one content allocation across
350            // every model instead of cloning the note body per task.
351            let text: Arc<str> = Arc::from(crate::curation::note_embedding_text_ref(note));
352            for (model_idx, model_name) in embed_model_names.iter().enumerate() {
353                let rt = runtime.clone();
354                let token = spec.token.clone();
355                let name = model_name.clone();
356                let text = Arc::clone(&text);
357                let ctx = usage_ctx.clone();
358                join_set.spawn(async move {
359                    let fut = rt.embed_document_with_model_outcome_for_token(
360                        &token,
361                        &name,
362                        text.as_ref(),
363                    );
364                    let result = match ctx {
365                        Some(ctx) => crate::usage::scope(ctx, fut).await,
366                        None => fut.await,
367                    };
368                    (content_group_idx, model_idx, result)
369                });
370            }
371        }
372
373        while let Some(joined) = join_set.join_next().await {
374            match joined {
375                Ok((content_group_idx, model_idx, Ok(outcome))) => {
376                    embedding_outcomes[content_group_idx][model_idx] = Some(outcome);
377                }
378                Ok((_, _, Err(e))) => {
379                    join_set.abort_all();
380                    return Err(e);
381                }
382                Err(join_err) => {
383                    join_set.abort_all();
384                    return Err(RuntimeError::Internal(format!(
385                        "embed task panicked: {join_err}"
386                    )));
387                }
388            }
389        }
390    }
391
392    // Preserve the report's logical note/model accounting even when identical
393    // siblings share one provider invocation.
394    for &content_group_idx in &note_content_groups {
395        for outcome in embedding_outcomes[content_group_idx].iter().flatten() {
396            embedding_truncation.observe(outcome);
397            if outcome.truncated {
398                tracing::warn!(
399                    model = %outcome.model_name,
400                    source_bytes = outcome.source_bytes,
401                    embedded_bytes = outcome.embedded_bytes,
402                    "atomic note embedding input truncated; full content will be stored unchanged"
403                );
404            }
405        }
406    }
407
408    // Reject any non-finite embedding value BEFORE any plan is built — same
409    // validation the canonical `VectorStore::insert` DML performs, applied
410    // here so a bad custom embedding provider never reaches the raw vector
411    // insert on this path (embed-first: no note, FTS document, or vector row
412    // has been written yet).
413    for outcomes_for_content in &embedding_outcomes {
414        for outcome in outcomes_for_content.iter().flatten() {
415            if let Some(idx) = non_finite_index(&outcome.vector) {
416                return Err(non_finite_vector_error(idx, outcome.vector[idx]));
417            }
418        }
419    }
420
421    // ---- 3. Build one AddNotePlan per spec (row + FTS + vector-insert
422    // statements), all pre-computed embeddings already in hand. ----
423    let mut plans: Vec<AtomicOpPlan> = Vec::with_capacity(notes.len());
424    for (note_idx, note) in notes.iter().enumerate() {
425        let outcomes_for_note = &embedding_outcomes[note_content_groups[note_idx]];
426        let mut statements = vec![PlanStatement {
427            statement: khive_db::stores::note::note_upsert_statement(note),
428            guard: Some(AffectedRowGuard::exactly(1)),
429        }];
430
431        if let Some(fault) = maybe_inject_fts_failure(&note.namespace, "fault-injected-fts") {
432            statements.push(fault);
433        } else {
434            // Delete-then-insert upsert — mirrors `text.rs`'s
435            // `upsert_document_dml` exactly, so a caller-supplied/reused note
436            // id never leaves a stale/duplicate FTS document behind (the
437            // note row itself is already an upsert via
438            // `note_upsert_statement`).
439            statements.push(PlanStatement {
440                statement: khive_db::stores::text::delete_document_statement(
441                    "fts_notes",
442                    &note.namespace,
443                    note.id,
444                ),
445                guard: None,
446            });
447            statements.push(PlanStatement {
448                statement: khive_db::stores::text::insert_document_statement(
449                    "fts_notes",
450                    &note_fts_document(note),
451                ),
452                guard: None,
453            });
454        }
455
456        if let Some(fault) = maybe_inject_vector_failure(&note.namespace, "fault-injected-vector") {
457            statements.push(fault);
458        } else {
459            for (model_name, outcome) in embed_model_names.iter().zip(outcomes_for_note) {
460                let outcome = outcome
461                    .as_ref()
462                    .expect("every model index observed exactly once");
463                let table = format!("vec_{}", crate::config::sanitize_key(model_name));
464                statements.extend(vector_insert_statements(
465                    &table,
466                    &note.namespace,
467                    note.id,
468                    "note.content",
469                    model_name,
470                    &outcome.vector,
471                    &format!("atomic-message-vec-{table}-{}", note.id),
472                ));
473            }
474        }
475
476        plans.push(AtomicOpPlan::AddNote(AddNotePlan {
477            note_id: note.id,
478            statements,
479            post_commit: PostCommitEffect::None,
480        }));
481    }
482
483    // ---- 4. One writer acquisition for the whole set. ----
484    match run_atomic_unit(runtime.sql().as_ref(), plans).await {
485        Ok(AtomicRunOutcome::Committed { .. }) => Ok((notes, embedding_truncation)),
486        Ok(AtomicRunOutcome::RolledBack {
487            failed_op_index,
488            failure,
489        }) => Err(RuntimeError::Internal(format!(
490            "atomic multi-note write rolled back at op {failed_op_index}: {failure:?}"
491        ))),
492        Err(e) => Err(RuntimeError::Storage(e.0)),
493    }
494}
495
496#[cfg(test)]
497mod tests {
498    use super::*;
499    use async_trait::async_trait;
500    use lattice_embed::{EmbedError, EmbeddingModel, EmbeddingService, MAX_TEXT_BYTES};
501
502    use khive_types::Namespace;
503
504    use crate::embedder_registry::EmbedderProvider;
505
506    const NAN_MODEL: &str = "atomic-message-nan-model";
507    const NAN_DIMS: usize = 4;
508
509    struct NanService;
510    #[async_trait]
511    impl EmbeddingService for NanService {
512        async fn embed(
513            &self,
514            texts: &[String],
515            _model: EmbeddingModel,
516        ) -> Result<Vec<Vec<f32>>, EmbedError> {
517            Ok(texts.iter().map(|_| vec![f32::NAN; NAN_DIMS]).collect())
518        }
519        fn supports_model(&self, _model: EmbeddingModel) -> bool {
520            true
521        }
522        fn name(&self) -> &'static str {
523            NAN_MODEL
524        }
525    }
526    struct NanProvider;
527    #[async_trait]
528    impl EmbedderProvider for NanProvider {
529        fn name(&self) -> &str {
530            NAN_MODEL
531        }
532        fn dimensions(&self) -> usize {
533            NAN_DIMS
534        }
535        async fn build(&self) -> RuntimeResult<std::sync::Arc<dyn EmbeddingService>> {
536            Ok(std::sync::Arc::new(NanService))
537        }
538    }
539
540    struct TruncationService;
541    #[async_trait]
542    impl EmbeddingService for TruncationService {
543        async fn embed(
544            &self,
545            texts: &[String],
546            _model: EmbeddingModel,
547        ) -> Result<Vec<Vec<f32>>, EmbedError> {
548            Ok(texts
549                .iter()
550                .map(|_| vec![0.5; EmbeddingModel::MultilingualE5Base.dimensions()])
551                .collect())
552        }
553        fn supports_model(&self, _model: EmbeddingModel) -> bool {
554            true
555        }
556        fn name(&self) -> &'static str {
557            "multilingual-e5-base"
558        }
559    }
560
561    struct TruncationProvider;
562    #[async_trait]
563    impl EmbedderProvider for TruncationProvider {
564        fn name(&self) -> &str {
565            "multilingual-e5-base"
566        }
567        fn dimensions(&self) -> usize {
568            EmbeddingModel::MultilingualE5Base.dimensions()
569        }
570        async fn build(&self) -> RuntimeResult<std::sync::Arc<dyn EmbeddingService>> {
571            Ok(std::sync::Arc::new(TruncationService))
572        }
573    }
574
575    const DEDUP_DIMS: usize = 4;
576
577    struct DedupService {
578        name: &'static str,
579    }
580
581    #[async_trait]
582    impl EmbeddingService for DedupService {
583        async fn embed(
584            &self,
585            texts: &[String],
586            _model: EmbeddingModel,
587        ) -> Result<Vec<Vec<f32>>, EmbedError> {
588            Ok(texts.iter().map(|_| vec![0.25; DEDUP_DIMS]).collect())
589        }
590
591        fn supports_model(&self, _model: EmbeddingModel) -> bool {
592            true
593        }
594
595        fn name(&self) -> &'static str {
596            self.name
597        }
598    }
599
600    struct DedupProvider {
601        name: &'static str,
602    }
603
604    #[async_trait]
605    impl EmbedderProvider for DedupProvider {
606        fn name(&self) -> &str {
607            self.name
608        }
609
610        fn dimensions(&self) -> usize {
611            DEDUP_DIMS
612        }
613
614        async fn build(&self) -> RuntimeResult<std::sync::Arc<dyn EmbeddingService>> {
615            Ok(std::sync::Arc::new(DedupService { name: self.name }))
616        }
617    }
618
619    /// A minimal note-write validator standing in for a pack's real one (e.g.
620    /// `khive-pack-comm`'s `derive_message_identity`): unconditionally stamps
621    /// `from_actor` with the caller's token-derived actor id, discarding
622    /// whatever the caller's own `properties` named for that key.
623    fn stamp_from_actor(
624        _kind: &str,
625        actor_id: &str,
626        properties: Option<Value>,
627    ) -> RuntimeResult<Option<Value>> {
628        let mut props = match properties {
629            Some(Value::Object(map)) => map,
630            _ => serde_json::Map::new(),
631        };
632        props.insert(
633            "from_actor".to_string(),
634            Value::String(actor_id.to_string()),
635        );
636        Ok(Some(Value::Object(props)))
637    }
638
639    async fn fts_row_count(runtime: &KhiveRuntime, namespace: &str) -> i64 {
640        let mut reader = runtime.sql().reader().await.expect("sql reader");
641        match reader
642            .query_scalar(SqlStatement {
643                sql: "SELECT COUNT(*) FROM fts_notes WHERE namespace = ?1".to_string(),
644                params: vec![SqlValue::Text(namespace.to_string())],
645                label: None,
646            })
647            .await
648            .expect("fts count query")
649        {
650            Some(SqlValue::Integer(n)) => n,
651            other => panic!("unexpected fts count result: {other:?}"),
652        }
653    }
654
655    async fn ann_write_log_count(runtime: &KhiveRuntime, namespace: &str, model: &str) -> i64 {
656        let mut reader = runtime.sql().reader().await.expect("sql reader");
657        match reader
658            .query_scalar(SqlStatement {
659                sql: "SELECT COUNT(*) FROM ann_write_log \
660                      WHERE namespace = ?1 AND embedding_model = ?2"
661                    .to_string(),
662                params: vec![
663                    SqlValue::Text(namespace.to_string()),
664                    SqlValue::Text(model.to_string()),
665                ],
666                label: None,
667            })
668            .await
669            .expect("ann_write_log count query")
670        {
671            Some(SqlValue::Integer(n)) => n,
672            other => panic!("unexpected ann_write_log count result: {other:?}"),
673        }
674    }
675
676    /// A custom embedding provider returning NaN must be rejected BEFORE any
677    /// plan is built — same validation `VectorStore::insert` performs on the
678    /// canonical `create_note_inner` path. No note row, FTS document, vector
679    /// row, or `ann_write_log` row may be committed.
680    #[tokio::test]
681    async fn create_notes_atomic_rejects_non_finite_embedding_before_any_write() {
682        let runtime = KhiveRuntime::memory().expect("in-memory runtime");
683        runtime.register_embedder(NanProvider);
684        let ns = "atomic-message-nan-test";
685        let token = runtime
686            .authorize(Namespace::parse(ns).unwrap())
687            .expect("authorize");
688
689        let result = create_notes_atomic(
690            &runtime,
691            vec![AtomicNoteSpec {
692                token: &token,
693                id: None,
694                kind: "observation",
695                name: None,
696                content: "nan embedding content",
697                properties: None,
698            }],
699        )
700        .await;
701
702        assert!(
703            result.is_err(),
704            "a non-finite embedding must be rejected; got {result:?}"
705        );
706
707        let alive = runtime
708            .list_notes(&token, Some("observation"), 100, 0)
709            .await
710            .expect("list_notes")
711            .into_iter()
712            .filter(|n| n.deleted_at.is_none())
713            .count();
714        assert_eq!(
715            alive, 0,
716            "no note row may be committed when an embedding is non-finite"
717        );
718
719        assert_eq!(
720            fts_row_count(&runtime, ns).await,
721            0,
722            "no FTS document may be committed when an embedding is non-finite"
723        );
724
725        let vs = runtime
726            .vectors_for_model(&token, NAN_MODEL)
727            .expect("vec store");
728        assert_eq!(
729            vs.count().await.expect("count"),
730            0,
731            "no vector row may be committed when an embedding is non-finite"
732        );
733
734        assert_eq!(
735            ann_write_log_count(&runtime, ns, NAN_MODEL).await,
736            0,
737            "no ann_write_log row may be committed when an embedding is non-finite"
738        );
739    }
740
741    #[tokio::test]
742    async fn create_notes_atomic_rejects_reserved_secret_gate_key() {
743        let runtime = KhiveRuntime::memory().expect("in-memory runtime");
744        let ns = "atomic-message-reserved-key-test";
745        let token = runtime
746            .authorize(Namespace::parse(ns).unwrap())
747            .expect("authorize");
748
749        let result = create_notes_atomic(
750            &runtime,
751            vec![AtomicNoteSpec {
752                token: &token,
753                id: None,
754                kind: "observation",
755                name: None,
756                content: "atomic message reservation target",
757                properties: Some(
758                    serde_json::json!({"khive:secret_gate": "exempted:content-sha256-manifest-v1"}),
759                ),
760            }],
761        )
762        .await;
763
764        let err = result.expect_err("caller-supplied reserved key must be rejected");
765        assert!(
766            matches!(err, RuntimeError::InvalidInput(ref msg) if msg.contains("khive:secret_gate")),
767            "unexpected error: {err:?}"
768        );
769
770        let alive = runtime
771            .list_notes(&token, Some("observation"), 100, 0)
772            .await
773            .expect("list_notes")
774            .into_iter()
775            .filter(|n| n.deleted_at.is_none())
776            .count();
777        assert_eq!(
778            alive, 0,
779            "no note row may be committed on reservation rejection"
780        );
781    }
782
783    #[tokio::test]
784    async fn create_notes_atomic_with_report_preserves_truncation_outcome() {
785        let runtime = KhiveRuntime::memory().expect("in-memory runtime");
786        runtime.register_embedder(TruncationProvider);
787        let token = runtime
788            .authorize(Namespace::parse("atomic-message-truncation-test").unwrap())
789            .expect("authorize");
790        let content = "x".repeat(MAX_TEXT_BYTES);
791
792        let (notes, report) = create_notes_atomic_with_report(
793            &runtime,
794            vec![AtomicNoteSpec {
795                token: &token,
796                id: None,
797                kind: "observation",
798                name: None,
799                content: &content,
800                properties: None,
801            }],
802        )
803        .await
804        .expect("atomic note write");
805
806        assert_eq!(notes.len(), 1);
807        assert_eq!(report.truncated, 1);
808        assert_eq!(
809            report.discarded_bytes,
810            "passage: ".len() as u64,
811            "E5 document-prefix reservation must be reflected in the returned report"
812        );
813    }
814
815    #[tokio::test]
816    async fn create_notes_atomic_reuses_identical_content_once_per_model() {
817        let runtime = KhiveRuntime::memory().expect("in-memory runtime");
818        for name in ["atomic-dedup-model-a", "atomic-dedup-model-b"] {
819            runtime.register_embedder(DedupProvider { name });
820        }
821        let outbound_token = runtime
822            .authorize(Namespace::parse("atomic-dedup-outbound").unwrap())
823            .expect("authorize outbound");
824        let inbound_token = runtime
825            .authorize(Namespace::parse("atomic-dedup-inbound").unwrap())
826            .expect("authorize inbound");
827        let content = "byte-identical outbound and inbound message content";
828
829        let usage = crate::usage::UsageContext::new();
830        let notes = crate::usage::scope(usage.clone(), async {
831            create_notes_atomic(
832                &runtime,
833                vec![
834                    AtomicNoteSpec {
835                        token: &outbound_token,
836                        id: None,
837                        kind: "observation",
838                        name: Some("shared subject"),
839                        content,
840                        properties: Some(serde_json::json!({"direction": "outbound"})),
841                    },
842                    AtomicNoteSpec {
843                        token: &inbound_token,
844                        id: None,
845                        kind: "observation",
846                        name: Some("shared subject"),
847                        content,
848                        properties: Some(serde_json::json!({"direction": "inbound"})),
849                    },
850                ],
851            )
852            .await
853        })
854        .await
855        .expect("atomic note pair");
856
857        assert_eq!(notes.len(), 2);
858        assert_eq!(notes[0].content, content);
859        assert_eq!(notes[1].content, content);
860        assert_eq!(
861            usage.snapshot()["embed_calls"],
862            2,
863            "two identical notes across two models must issue one embed per model"
864        );
865        for model in ["atomic-dedup-model-a", "atomic-dedup-model-b"] {
866            for (direction, token) in [("outbound", &outbound_token), ("inbound", &inbound_token)] {
867                assert_eq!(
868                    runtime
869                        .vectors_for_model(token, model)
870                        .expect("vector store")
871                        .count()
872                        .await
873                        .expect("vector count"),
874                    1,
875                    "the {direction} note must retain its vector row for {model}"
876                );
877            }
878        }
879    }
880
881    /// Calling `create_notes_atomic` twice with the SAME caller-supplied id
882    /// (different content each time) must leave exactly one FTS document for
883    /// that id, reflecting the latest content — not an append of two
884    /// documents (the note row is already an upsert; the FTS half must match).
885    #[tokio::test]
886    async fn create_notes_atomic_upserts_fts_document_for_reused_note_id() {
887        let runtime = KhiveRuntime::memory().expect("in-memory runtime");
888        let ns = "atomic-message-fts-upsert-test";
889        let token = runtime
890            .authorize(Namespace::parse(ns).unwrap())
891            .expect("authorize");
892        let id = Uuid::new_v4();
893
894        create_notes_atomic(
895            &runtime,
896            vec![AtomicNoteSpec {
897                token: &token,
898                id: Some(id),
899                kind: "observation",
900                name: None,
901                content: "first content",
902                properties: None,
903            }],
904        )
905        .await
906        .expect("first write with supplied id");
907
908        create_notes_atomic(
909            &runtime,
910            vec![AtomicNoteSpec {
911                token: &token,
912                id: Some(id),
913                kind: "observation",
914                name: None,
915                content: "second content replacing the first",
916                properties: None,
917            }],
918        )
919        .await
920        .expect("second write reusing the same id");
921
922        assert_eq!(
923            fts_row_count(&runtime, ns).await,
924            1,
925            "exactly one FTS document may exist for the reused id"
926        );
927
928        let text = runtime.text_for_notes(&token).expect("text store");
929        let doc = text
930            .get_document(ns, id)
931            .await
932            .expect("get_document")
933            .expect("document exists for the reused id");
934        assert!(
935            doc.body.contains("second content"),
936            "the surviving FTS document must reflect the latest content; got {:?}",
937            doc.body
938        );
939    }
940
941    /// ADR-124 regression: `create_notes_atomic_with_report` must run
942    /// `derive_note_write_properties` per spec, the same as every other
943    /// note-write site (`operations.rs`'s create funnel,
944    /// `atomic_prepare::prepare_add_note`). A caller holding a valid token
945    /// for one actor must not be able to plant an arbitrary `from_actor` (or
946    /// any other owner-established property) via `AtomicNoteSpec::properties`
947    /// — the installed validator's derived value must win.
948    ///
949    /// This test fails if the `derive_note_write_properties` call in
950    /// `create_notes_atomic_with_report`'s spec loop is removed: verified by
951    /// deleting that call in a scratch copy of this file and re-running —
952    /// the assertion below turns red because the stored `from_actor` reverts
953    /// to the forged `"forged-actor"` value instead of the token's actor id.
954    #[tokio::test]
955    async fn create_notes_atomic_derives_from_actor_overwriting_a_forged_value() {
956        let runtime = KhiveRuntime::memory().expect("in-memory runtime");
957        runtime.install_note_write_validator(std::sync::Arc::new(stamp_from_actor));
958        let ns = "atomic-message-identity-guard-test";
959        let token = runtime
960            .authorize(Namespace::parse(ns).unwrap())
961            .expect("authorize");
962        let true_actor = token.actor().id.clone();
963
964        let notes = create_notes_atomic(
965            &runtime,
966            vec![AtomicNoteSpec {
967                token: &token,
968                id: None,
969                kind: "observation",
970                name: None,
971                content: "atomic writer identity guard probe",
972                properties: Some(serde_json::json!({"from_actor": "forged-actor"})),
973            }],
974        )
975        .await
976        .expect("atomic note write must succeed — the guard derives, it does not refuse");
977
978        assert_eq!(notes.len(), 1);
979        assert_eq!(
980            notes[0]
981                .properties
982                .as_ref()
983                .and_then(|p| p.get("from_actor"))
984                .and_then(|v| v.as_str()),
985            Some(true_actor.as_str()),
986            "the atomic writer must store the token-derived from_actor, not the \
987             caller-supplied forged value"
988        );
989    }
990}