Skip to main content

khive_runtime/operations/
delete_restore.rs

1use chrono::Utc;
2use uuid::Uuid;
3
4use khive_db::stores::entity::entity_hard_delete_statement;
5use khive_db::stores::event::hard_delete_lineage_warning_statements;
6use khive_db::stores::graph::purge_incident_edges_statement;
7use khive_db::stores::note::note_hard_delete_statement;
8use khive_db::stores::text::insert_document_statements;
9use khive_storage::types::{DeleteMode, SqlStatement, SqlValue};
10use khive_storage::{AttachmentSubstrate, Edge, Entity};
11use khive_types::{EventKind, SubstrateKind};
12
13use crate::atomic_plan::{
14    AffectedRowGuard, DeletePlan, PlanStatement, PostCommitEffect, UpdatePlan,
15};
16use crate::atomic_runner::{run_atomic_unit, AtomicOpFailure, AtomicOpPlan, AtomicRunOutcome};
17use crate::curation::entity_fts_document;
18use crate::error::{RuntimeError, RuntimeResult};
19use crate::runtime::{KhiveRuntime, NamespaceToken};
20
21#[cfg(any(test, feature = "fault-injection"))]
22use super::{consume_fault, FTS_FAIL_NS, ROLLBACK_CLEANUP_FAIL_NS};
23use super::{
24    legacy_post_commit_result, live_merged_entity_refused, merge_tombstone_restore_refused,
25    record_post_commit_degradation, restore_reindex_failed, PostCommitDegradation,
26};
27
28impl KhiveRuntime {
29    /// Hard-delete a single graph node (entity, note, or edge-as-node row) AND purge its
30    /// incident edges in ONE write transaction — closes a race where a concurrent guarded
31    /// write could insert a fresh edge against the endpoint between two separately
32    /// committed calls; see docs/operations.md#atomic_hard_delete_with_edge_purge.
33    ///
34    /// `row_statement` is the exact hard-delete `DELETE` for the target row
35    /// (entity, note, or edge). Before the incident-edge purge, the same plan
36    /// appends any ADR-002 lineage-loss warnings from the still-present edge
37    /// rows, so the warning payload and cascade commit or roll back together.
38    /// Returns `Ok(true)` if the row was deleted, `Ok(false)` if it no longer
39    /// existed (lost a race with a concurrent delete of the same row).
40    pub(super) async fn atomic_hard_delete_with_edge_purge(
41        &self,
42        row_statement: SqlStatement,
43        node_id: Uuid,
44        namespace: &str,
45        actor: &str,
46        substrate: SubstrateKind,
47    ) -> RuntimeResult<bool> {
48        let mut statements = vec![PlanStatement {
49            statement: row_statement,
50            guard: Some(AffectedRowGuard::exactly(1)),
51        }];
52        if matches!(substrate, SubstrateKind::Entity | SubstrateKind::Note) {
53            statements.push(PlanStatement {
54                statement: khive_db::stores::attachment::delete_record_attachments_statement(
55                    node_id,
56                    if substrate == SubstrateKind::Entity {
57                        AttachmentSubstrate::Entity
58                    } else {
59                        AttachmentSubstrate::Note
60                    },
61                ),
62                guard: None,
63            });
64        }
65        statements.extend(
66            hard_delete_lineage_warning_statements(namespace, actor, node_id, substrate)
67                .into_iter()
68                .map(|statement| PlanStatement {
69                    statement,
70                    guard: None,
71                }),
72        );
73        statements.push(PlanStatement {
74            statement: purge_incident_edges_statement(node_id),
75            guard: None,
76        });
77        let plan = AtomicOpPlan::Delete(DeletePlan {
78            target_id: node_id,
79            statements,
80            post_commit: PostCommitEffect::None,
81        });
82        match run_atomic_unit(self.sql().as_ref(), vec![plan]).await {
83            Ok(AtomicRunOutcome::Committed { .. }) => Ok(true),
84            Ok(AtomicRunOutcome::RolledBack {
85                failure: AtomicOpFailure::NoteConflict(conflict),
86                ..
87            }) => Err(conflict.into_error().into()),
88            Ok(AtomicRunOutcome::RolledBack {
89                failure: AtomicOpFailure::EntityConflict(conflict),
90                ..
91            }) => Err(conflict.into_error().into()),
92            Ok(AtomicRunOutcome::RolledBack {
93                failure: AtomicOpFailure::GuardFailed { .. },
94                ..
95            }) => Ok(false),
96            Ok(AtomicRunOutcome::RolledBack {
97                failure: AtomicOpFailure::SqlError { message, .. },
98                ..
99            }) => Err(RuntimeError::Internal(format!(
100                "hard delete + edge purge for {node_id} failed: {message}"
101            ))),
102            // Only canonical change-set plans compare preimages or owner snapshots.
103            Ok(AtomicRunOutcome::RolledBack {
104                failure:
105                    failure @ (AtomicOpFailure::PreimageMismatch { .. }
106                    | AtomicOpFailure::OwnerSnapshotConflict { .. }),
107                ..
108            }) => Err(RuntimeError::Internal(format!(
109                "hard delete + edge purge for {node_id}: unexpected canonical failure {failure:?}"
110            ))),
111            Err(e) => Err(RuntimeError::Internal(format!(
112                "hard delete + edge purge for {node_id}: atomic unit seam failure: {}",
113                e.0
114            ))),
115        }
116    }
117
118    /// Restore an entity tombstone owned by the caller's primary namespace.
119    ///
120    /// The restore is guarded by both the tombstone's id/namespace and the
121    /// current uniqueness state, so a caller cannot resurrect over a newer
122    /// live record. Indexes are rebuilt only after the row restore commits.
123    pub async fn restore_entity(
124        &self,
125        token: &NamespaceToken,
126        id: Uuid,
127    ) -> RuntimeResult<Option<(Entity, bool)>> {
128        let Some(entity) = self
129            .entities(token)?
130            .get_entity_including_deleted(id)
131            .await?
132        else {
133            return Ok(None);
134        };
135        if entity.namespace != token.namespace().as_str() {
136            return Ok(None);
137        }
138        // A merge tombstone is not a plain soft delete: the source row carries
139        // merge provenance and its content already lives on the kept entity.
140        // Clearing only `deleted_at` would bring the source back as a live
141        // duplicate that still claims to have been merged. Refuse and name
142        // the kept id; restore does not undo a merge.
143        //
144        // The merge check runs before the already-live short cut on purpose:
145        // a live row that still carries `merged_into` is what an earlier
146        // restore left behind before this guard existed, and answering it
147        // "already live" would hide the invariant violation from the one
148        // caller who is looking at the row. Name it instead.
149        if let Some(kept_id) = entity.merged_into {
150            if entity.deleted_at.is_none() {
151                return Err(live_merged_entity_refused(id, kept_id));
152            }
153            return Err(merge_tombstone_restore_refused(id, kept_id));
154        }
155        if entity.deleted_at.is_none() {
156            return Ok(Some((entity, false)));
157        }
158        let updated_at =
159            Utc::now()
160                .timestamp_micros()
161                .max(entity.updated_at.checked_add(1).ok_or_else(|| {
162                    RuntimeError::Internal(format!(
163                        "entity {id} updated_at is already at i64::MAX and cannot advance"
164                    ))
165                })?);
166        let mut restored = entity;
167        restored.deleted_at = None;
168        restored.updated_at = updated_at;
169        restored.version = restored
170            .version
171            .checked_add(1)
172            .ok_or_else(|| RuntimeError::InvalidInput("entity version overflow".into()))?;
173        let mut statements = vec![PlanStatement {
174            statement: SqlStatement {
175                sql: "UPDATE entities SET deleted_at=NULL, updated_at=?1, version=version+1 \
176                      WHERE id=?2 AND namespace=?3 AND deleted_at IS NOT NULL AND version=?4"
177                    .into(),
178                params: vec![
179                    SqlValue::Integer(updated_at),
180                    SqlValue::Text(id.to_string()),
181                    SqlValue::Text(token.namespace().as_str().to_owned()),
182                    SqlValue::Integer(restored.version - 1),
183                ],
184                label: Some("entity-restore".into()),
185            },
186            guard: Some(AffectedRowGuard::exactly(1)),
187        }];
188        // The soft delete removed the FTS row, so the text index is published
189        // in the same unit as the row: a live row that search cannot find is
190        // not a state this verb can leave behind. Order-sensitive pair — see
191        // `insert_document_statements`'s adjacency contract.
192        for statement in khive_db::stores::text::delete_document_statements(
193            "fts_entities",
194            &restored.namespace,
195            id,
196        )
197        .into_iter()
198        .chain(insert_document_statements(
199            "fts_entities",
200            &entity_fts_document(&restored),
201        )) {
202            statements.push(PlanStatement {
203                statement,
204                guard: None,
205            });
206        }
207        let plan = AtomicOpPlan::Update(Box::new(UpdatePlan {
208            graph_effects: Vec::new(),
209            target_id: id,
210            statements,
211            post_commit: PostCommitEffect::None,
212            edge_natural_key: None,
213            idempotent_noop: false,
214            entity_guard: None,
215            note_guard: None,
216            note_vector_purge: None,
217            note_embedding_inheritance: None,
218        }));
219        match run_atomic_unit(self.sql().as_ref(), vec![plan]).await {
220            Ok(AtomicRunOutcome::Committed { .. }) => {
221                // Embeddings are rebuilt after the commit; the row and its
222                // text index are already live, so a failure here names that.
223                #[cfg(any(test, feature = "fault-injection"))]
224                if consume_fault(&FTS_FAIL_NS, &restored.namespace) {
225                    return Err(restore_reindex_failed(
226                        "entity",
227                        id,
228                        RuntimeError::Internal("injected FTS failure".to_string()),
229                    ));
230                }
231                self.reindex_entity(token, &restored)
232                    .await
233                    .map_err(|e| restore_reindex_failed("entity", id, e))?;
234                Ok(Some((restored, true)))
235            }
236            Ok(AtomicRunOutcome::RolledBack { failure, .. }) => Err(RuntimeError::Internal(
237                format!("entity restore rolled back: {failure:?}"),
238            )),
239            Err(error) => Err(RuntimeError::Storage(error.0)),
240        }
241    }
242
243    /// Restore an edge tombstone owned by the caller's primary namespace.
244    pub async fn restore_edge(
245        &self,
246        token: &NamespaceToken,
247        id: Uuid,
248    ) -> RuntimeResult<Option<(Edge, bool)>> {
249        let Some(edge) = self.get_edge_including_deleted(token, id).await? else {
250            return Ok(None);
251        };
252        if edge.namespace != token.namespace().as_str() {
253            return Ok(None);
254        }
255        if edge.deleted_at.is_none() {
256            return Ok(Some((edge, false)));
257        }
258        let both_memory_at_check = match (
259            self.resolve_by_id_including_deleted(token, edge.source_id)
260                .await?,
261            self.resolve_by_id_including_deleted(token, edge.target_id)
262                .await?,
263        ) {
264            (Some(crate::Resolved::Note(source)), Some(crate::Resolved::Note(target))) => {
265                let both_memory = source.kind == "memory" && target.kind == "memory";
266                Self::ensure_memory_note_edge_namespace(
267                    token,
268                    "restore",
269                    &edge.namespace,
270                    (edge.source_id, edge.target_id),
271                    edge.relation,
272                    &source,
273                    &target,
274                )?;
275                both_memory
276            }
277            _ => false,
278        };
279        let updated_at = Utc::now();
280        let plan = AtomicOpPlan::Update(Box::new(UpdatePlan {
281            graph_effects: Vec::new(),
282            target_id: id,
283            statements: vec![PlanStatement {
284                statement: SqlStatement {
285                    sql: "UPDATE graph_edges SET deleted_at=NULL, updated_at=?1 \
286                          WHERE id=?2 AND namespace=?3 AND deleted_at IS NOT NULL \
287                            AND ((?4 = 0 AND ( \
288                                  NOT EXISTS (SELECT 1 FROM notes AS source_memory \
289                                       WHERE source_memory.id=graph_edges.source_id \
290                                         AND source_memory.kind='memory') \
291                                  OR NOT EXISTS (SELECT 1 FROM notes AS target_memory \
292                                       WHERE target_memory.id=graph_edges.target_id \
293                                         AND target_memory.kind='memory') \
294                                 )) \
295                                 OR EXISTS (SELECT 1 FROM notes AS source_match \
296                                      WHERE source_match.id=graph_edges.source_id \
297                                        AND source_match.namespace=graph_edges.namespace \
298                                        AND source_match.kind='memory' \
299                                        AND source_match.deleted_at IS NULL))"
300                        .into(),
301                    params: vec![
302                        SqlValue::Integer(updated_at.timestamp_micros()),
303                        SqlValue::Text(id.to_string()),
304                        SqlValue::Text(edge.namespace.clone()),
305                        SqlValue::Integer(if both_memory_at_check { 1 } else { 0 }),
306                    ],
307                    label: Some("edge-restore".into()),
308                },
309                guard: Some(AffectedRowGuard::exactly(1)),
310            }],
311            post_commit: PostCommitEffect::None,
312            edge_natural_key: None,
313            idempotent_noop: false,
314            entity_guard: None,
315            note_guard: None,
316            note_vector_purge: None,
317            note_embedding_inheritance: None,
318        }));
319        #[cfg(test)]
320        memory_edge_namespace_restore_tests::before_writer(id).await;
321        match run_atomic_unit(self.sql().as_ref(), vec![plan]).await {
322            Ok(AtomicRunOutcome::Committed { .. }) => {
323                let mut restored = edge;
324                restored.deleted_at = None;
325                restored.updated_at = updated_at;
326                Ok(Some((restored, true)))
327            }
328            Ok(AtomicRunOutcome::RolledBack {
329                failure:
330                    AtomicOpFailure::GuardFailed {
331                        statement_label: Some(label),
332                        expected,
333                        observed: 0,
334                    },
335                ..
336            }) if label == "edge-restore" && expected == AffectedRowGuard::exactly(1) => {
337                Err(RuntimeError::InvalidInput(
338                    "edge restore preconditions changed before commit; retry against the current edge and endpoint state".into(),
339                ))
340            }
341            Ok(AtomicRunOutcome::RolledBack { failure, .. }) => Err(RuntimeError::Internal(
342                format!("edge restore rolled back: {failure:?}"),
343            )),
344            Err(error) => Err(RuntimeError::Storage(error.0)),
345        }
346    }
347
348    /// Soft-delete or hard-delete a note by ID.
349    ///
350    /// On hard delete, cascades to remove all incident edges (both inbound and
351    /// outbound) and cleans up FTS and vector indexes, preventing dangling
352    /// references for `annotates` edges that target this note.
353    /// Soft delete also cleans FTS and vector indexes; edges are left in place.
354    ///
355    /// UUID v4 is globally unique: no namespace filter on by-ID ops.
356    /// Cascade and index cleanup target the RECORD's stored namespace, not the caller token's.
357    /// Returns `Ok(false)` if the note does not exist.
358    pub async fn delete_note(
359        &self,
360        token: &NamespaceToken,
361        id: Uuid,
362        hard: bool,
363    ) -> RuntimeResult<bool> {
364        let (deleted, degradations) = self
365            .delete_note_with_post_commit_report(token, id, hard)
366            .await?;
367        legacy_post_commit_result("delete_note", id, deleted, degradations)
368    }
369
370    /// Delete the note and return diagnostics for any failed work after the
371    /// row change committed. A degradation is non-retryable: callers must not
372    /// repeat a create or delete because an index or telemetry leg failed.
373    pub async fn delete_note_with_post_commit_report(
374        &self,
375        token: &NamespaceToken,
376        id: Uuid,
377        hard: bool,
378    ) -> RuntimeResult<(bool, Vec<PostCommitDegradation>)> {
379        let note_store = self.notes(token)?;
380        let note = if hard {
381            match note_store.get_note_including_deleted(id).await? {
382                Some(n) => n,
383                None => return Ok((false, Vec::new())),
384            }
385        } else {
386            match note_store.get_note(id).await? {
387                Some(n) => n,
388                None => return Ok((false, Vec::new())),
389            }
390        };
391        if let Some(error) = self.stream_member_error(&note).await? {
392            return Err(error);
393        }
394        let mode = if hard {
395            DeleteMode::Hard
396        } else {
397            DeleteMode::Soft
398        };
399
400        // Route index cleanup through the RECORD's namespace, not the caller's.
401        let record_tok = token.with_namespace(
402            khive_types::Namespace::parse(&note.namespace)
403                .map_err(|e| RuntimeError::Internal(format!("note namespace invalid: {e}")))?,
404        );
405        let record_ns = note.namespace.clone();
406        let actor = format!("{}:{}", token.actor().kind, token.actor().id);
407
408        // On hard delete, the row delete and the incident-edge cascade (including
409        // already-soft-deleted edges) run as ONE write transaction: see
410        // `atomic_hard_delete_with_edge_purge`. Index cleanup follows the
411        // commit; it is best-effort and idempotent, unlike the row/edge pair.
412        let deleted = if hard {
413            self.atomic_hard_delete_with_edge_purge(
414                note_hard_delete_statement(id),
415                id,
416                &record_ns,
417                &actor,
418                SubstrateKind::Note,
419            )
420            .await?
421        } else {
422            note_store.delete_note(id, mode).await?
423        };
424        let mut degradations = Vec::new();
425        if deleted {
426            let fts_result = match self.text_for_notes(&record_tok) {
427                Ok(store) => store
428                    .delete_document(&record_ns, id)
429                    .await
430                    .map_err(RuntimeError::from),
431                Err(error) => Err(error),
432            };
433            if let Err(error) = fts_result {
434                record_post_commit_degradation(
435                    &mut degradations,
436                    "delete_note",
437                    id,
438                    "fts_cleanup",
439                    error,
440                );
441            }
442            // Try every model even when FTS or another model failed.
443            for model_name in self.registered_embedding_model_names() {
444                let vector_result = match self.vectors_for_model(&record_tok, &model_name) {
445                    Ok(store) => store.delete(id).await.map_err(RuntimeError::from),
446                    Err(error) => Err(error),
447                };
448                if let Err(error) = vector_result {
449                    record_post_commit_degradation(
450                        &mut degradations,
451                        "delete_note",
452                        id,
453                        "vector_cleanup",
454                        format!("{model_name}: {error}"),
455                    );
456                }
457            }
458            let event = khive_storage::event::Event::new(
459                record_ns.clone(),
460                "delete",
461                EventKind::NoteDeleted,
462                SubstrateKind::Note,
463                "",
464            )
465            .with_target(id)
466            .with_payload(serde_json::json!({"id": id, "namespace": record_ns, "hard": hard}));
467            let event_result = match self.events(&record_tok) {
468                Ok(store) => store.append_event(event).await.map_err(RuntimeError::from),
469                Err(error) => Err(error),
470            };
471            if let Err(error) = event_result {
472                record_post_commit_degradation(
473                    &mut degradations,
474                    "delete_note",
475                    id,
476                    "event_append",
477                    error,
478                );
479            }
480            // A soft OR hard delete removes the note's vectors/FTS document
481            // above: any pack-owned vector-derived cache (e.g.
482            // khive-pack-memory's warm ANN index) needs to know the corpus
483            // changed, reached via this generic hook so khive-runtime never
484            // takes a dependency on khive-pack-memory. No-op when no pack has
485            // installed a hook.
486            self.fire_note_mutation_hook(&note.kind, id).await;
487        }
488        Ok((deleted, degradations))
489    }
490
491    /// Row-first compensating delete for rolling back a partially-written note
492    /// (e.g. `dual_write_message` rollback after a later delivery step fails).
493    /// Unlike [`KhiveRuntime::delete_note`], which cleans up graph/FTS/
494    /// vector indexes *before* removing the row, this removes the row first so
495    /// that a cleanup failure afterward cannot leave the compensated note live.
496    ///
497    /// Returns `Ok(())` once the row is gone (whether or not cleanup fully
498    /// succeeded). Returns `Err(RuntimeError::Internal)` naming the failed
499    /// cleanup legs when row removal succeeded but cleanup did not — the
500    /// message is gone, but stale index entries may remain and should be
501    /// surfaced to the caller rather than silently discarded.
502    ///
503    /// Returns `Ok(())` immediately, with no cleanup attempted, if the note
504    /// does not exist (nothing to compensate).
505    ///
506    /// Not a general-purpose replacement for `delete_note(..., hard=true)`:
507    /// normal hard delete still needs cleanup-first semantics (no dangling
508    /// references) since a caller-visible error there should not remove the row.
509    pub async fn delete_note_row_first_for_compensation(
510        &self,
511        token: &NamespaceToken,
512        id: Uuid,
513    ) -> RuntimeResult<()> {
514        let note_store = self.notes(token)?;
515        let Some(note) = note_store.get_note_including_deleted(id).await? else {
516            return Ok(());
517        };
518        let record_tok = NamespaceToken::for_namespace(
519            khive_types::Namespace::parse(&note.namespace)
520                .map_err(|e| RuntimeError::Internal(format!("note namespace invalid: {e}")))?,
521        );
522        let record_ns = note.namespace.clone();
523
524        // Critical ordering: remove the row before any cleanup that can fail.
525        note_store.delete_note(id, DeleteMode::Hard).await?;
526
527        #[cfg(any(test, feature = "fault-injection"))]
528        {
529            let armed = ROLLBACK_CLEANUP_FAIL_NS.lock().unwrap().take();
530            if armed.as_deref() == Some(record_ns.as_str()) {
531                return Err(RuntimeError::Internal(
532                    "row removed but compensation cleanup failed: injected=true".to_string(),
533                ));
534            }
535        }
536
537        let mut cleanup_errors = Vec::new();
538        if let Err(e) = self.graph(&record_tok)?.purge_incident_edges(id).await {
539            cleanup_errors.push(format!("graph={e}"));
540        }
541        if let Err(e) = self
542            .text_for_notes(&record_tok)?
543            .delete_document(&record_ns, id)
544            .await
545        {
546            cleanup_errors.push(format!("fts={e}"));
547        }
548        for model_name in self.registered_embedding_model_names() {
549            if let Err(e) = self
550                .vectors_for_model(&record_tok, &model_name)?
551                .delete(id)
552                .await
553            {
554                cleanup_errors.push(format!("vector[{model_name}]={e}"));
555            }
556        }
557        if cleanup_errors.is_empty() {
558            Ok(())
559        } else {
560            Err(RuntimeError::Internal(format!(
561                "row removed but compensation cleanup failed: {}",
562                cleanup_errors.join("; ")
563            )))
564        }
565    }
566
567    /// Soft-delete or hard-delete an entity by ID (soft delete by default).
568    ///
569    /// On hard delete, cascades to remove all incident edges (both inbound and
570    /// outbound) to prevent dangling references. Soft delete also cleans FTS
571    /// and vector indexes; edges are left in place.
572    /// Routed attachment cleanup is performed by the registry after ownership resolution.
573    ///
574    /// UUID v4 is globally unique: no namespace filter on by-ID ops.
575    pub async fn delete_entity(
576        &self,
577        token: &NamespaceToken,
578        id: Uuid,
579        hard: bool,
580    ) -> RuntimeResult<bool> {
581        let (deleted, degradations) = self
582            .delete_entity_with_post_commit_report(token, id, hard)
583            .await?;
584        legacy_post_commit_result("delete_entity", id, deleted, degradations)
585    }
586
587    /// The committed delete together with non-retryable index/telemetry errors.
588    pub async fn delete_entity_with_post_commit_report(
589        &self,
590        token: &NamespaceToken,
591        id: Uuid,
592        hard: bool,
593    ) -> RuntimeResult<(bool, Vec<PostCommitDegradation>)> {
594        let entity = if hard {
595            match self
596                .entities(token)?
597                .get_entity_including_deleted(id)
598                .await?
599            {
600                Some(e) => e,
601                None => return Ok((false, Vec::new())),
602            }
603        } else {
604            match self.entities(token)?.get_entity(id).await? {
605                Some(e) => e,
606                None => return Ok((false, Vec::new())),
607            }
608        };
609        let mode = if hard {
610            DeleteMode::Hard
611        } else {
612            DeleteMode::Soft
613        };
614
615        // Route cascade and index cleanup through the RECORD's namespace, not the caller's.
616        let record_tok = token.with_namespace(
617            khive_types::Namespace::parse(&entity.namespace)
618                .map_err(|e| RuntimeError::Internal(format!("entity namespace invalid: {e}")))?,
619        );
620        let actor = format!("{}:{}", token.actor().kind, token.actor().id);
621
622        // On hard delete, the row delete and the incident-edge cascade (including
623        // already-soft-deleted edges) run as ONE write transaction: see
624        // `atomic_hard_delete_with_edge_purge`. Index cleanup follows the
625        // commit; it is best-effort and idempotent, unlike the row/edge pair.
626        let deleted = if hard {
627            // Cross-backend attachment cleanup requires the registry's ownership check.
628            self.atomic_hard_delete_with_edge_purge(
629                entity_hard_delete_statement(id),
630                id,
631                &entity.namespace,
632                &actor,
633                SubstrateKind::Entity,
634            )
635            .await?
636        } else {
637            self.entities(token)?.delete_entity(id, mode).await?
638        };
639        let mut degradations = Vec::new();
640        if deleted {
641            let ns = entity.namespace.clone();
642            let fts_result = match self.text(&record_tok) {
643                Ok(store) => store
644                    .delete_document(&ns, id)
645                    .await
646                    .map_err(RuntimeError::from),
647                Err(error) => Err(error),
648            };
649            if let Err(error) = fts_result {
650                record_post_commit_degradation(
651                    &mut degradations,
652                    "delete_entity",
653                    id,
654                    "fts_cleanup",
655                    error,
656                );
657            }
658            for model_name in self.registered_embedding_model_names() {
659                let vector_result = match self.vectors_for_model(&record_tok, &model_name) {
660                    Ok(store) => store.delete(id).await.map_err(RuntimeError::from),
661                    Err(error) => Err(error),
662                };
663                if let Err(error) = vector_result {
664                    record_post_commit_degradation(
665                        &mut degradations,
666                        "delete_entity",
667                        id,
668                        "vector_cleanup",
669                        format!("{model_name}: {error}"),
670                    );
671                }
672            }
673            let event = khive_storage::event::Event::new(
674                ns.clone(),
675                "delete",
676                EventKind::EntityDeleted,
677                SubstrateKind::Entity,
678                "",
679            )
680            .with_target(id)
681            .with_payload(serde_json::json!({"id": id, "namespace": ns, "hard": hard}));
682            let event_result = match self.events(&record_tok) {
683                Ok(store) => store.append_event(event).await.map_err(RuntimeError::from),
684                Err(error) => Err(error),
685            };
686            if let Err(error) = event_result {
687                record_post_commit_degradation(
688                    &mut degradations,
689                    "delete_entity",
690                    id,
691                    "event_append",
692                    error,
693                );
694            }
695        }
696        Ok((deleted, degradations))
697    }
698
699    pub(crate) async fn delete_entity_attachments_on_core(&self, id: Uuid) -> RuntimeResult<bool> {
700        let core = self.core();
701        drop(core.attachments()?);
702        let statement = khive_db::stores::attachment::delete_record_attachments_statement(
703            id,
704            AttachmentSubstrate::Entity,
705        );
706        Ok(core.sql().writer().await?.execute(statement).await? > 0)
707    }
708}
709#[cfg(test)]
710pub(super) mod memory_edge_namespace_restore_tests {
711    use crate::{KhiveRuntime, Resolved, RuntimeError};
712    use khive_storage::{Note, SqlStatement, SqlValue};
713    use khive_types::{EdgeRelation, Namespace};
714    use std::sync::Mutex;
715    use std::time::Duration;
716    use tokio::sync::oneshot;
717    use uuid::Uuid;
718
719    struct BeforeWriterHook {
720        edge_id: Uuid,
721        entered: oneshot::Sender<()>,
722        release: Option<oneshot::Receiver<()>>,
723    }
724
725    static BEFORE_WRITER_HOOKS: Mutex<Vec<BeforeWriterHook>> = Mutex::new(Vec::new());
726
727    fn install_before_writer_hook(
728        edge_id: Uuid,
729        entered: oneshot::Sender<()>,
730        release: Option<oneshot::Receiver<()>>,
731    ) {
732        let mut hooks = BEFORE_WRITER_HOOKS.lock().unwrap();
733        assert!(!hooks.iter().any(|hook| hook.edge_id == edge_id));
734        hooks.push(BeforeWriterHook {
735            edge_id,
736            entered,
737            release,
738        });
739    }
740
741    fn remove_before_writer_hook(edge_id: Uuid) {
742        BEFORE_WRITER_HOOKS
743            .lock()
744            .unwrap()
745            .retain(|hook| hook.edge_id != edge_id);
746    }
747
748    pub(super) async fn before_writer(edge_id: Uuid) {
749        let hook = {
750            let mut hooks = BEFORE_WRITER_HOOKS.lock().unwrap();
751            hooks
752                .iter()
753                .position(|hook| hook.edge_id == edge_id)
754                .map(|index| hooks.remove(index))
755        };
756        if let Some(hook) = hook {
757            let _ = hook.entered.send(());
758            if let Some(release) = hook.release {
759                let _ = release.await;
760            }
761        }
762    }
763
764    fn fixture() -> (KhiveRuntime, tempfile::TempDir) {
765        let dir = tempfile::tempdir().unwrap();
766        let runtime = KhiveRuntime::new_for_test(crate::RuntimeConfig {
767            db_path: Some(dir.path().join("restore-edge-memory-namespace.db")),
768            embedding_model: None,
769            additional_embedding_models: Vec::new(),
770            ..Default::default()
771        })
772        .unwrap();
773        (runtime, dir)
774    }
775
776    async fn execute_private_write(runtime: &KhiveRuntime, statement: SqlStatement) -> u64 {
777        let result = runtime
778            .sql()
779            .atomic_unit(Box::new(move |writer| {
780                Box::pin(async move {
781                    let affected = writer.execute(statement).await?;
782                    Ok(Box::new(affected) as Box<dyn std::any::Any + Send>)
783                })
784            }))
785            .await
786            .unwrap();
787        *result.downcast::<u64>().unwrap()
788    }
789
790    async fn prepare_tombstone(
791        runtime: &KhiveRuntime,
792        edge_id: Uuid,
793        from_namespace: &str,
794        stored_namespace: &str,
795    ) -> u64 {
796        let statement = SqlStatement {
797            sql: "UPDATE graph_edges SET namespace=?1, deleted_at=?2 \
798                  WHERE id=?3 AND namespace=?4 AND deleted_at IS NULL"
799                .into(),
800            params: vec![
801                SqlValue::Text(stored_namespace.into()),
802                SqlValue::Integer(chrono::Utc::now().timestamp_micros()),
803                SqlValue::Text(edge_id.to_string()),
804                SqlValue::Text(from_namespace.into()),
805            ],
806            label: Some("test-prepare-restore-tombstone".into()),
807        };
808        execute_private_write(runtime, statement).await
809    }
810
811    async fn move_memory_source_namespace(
812        runtime: &KhiveRuntime,
813        source_id: Uuid,
814        from_namespace: &str,
815        to_namespace: &str,
816    ) -> u64 {
817        let statement = SqlStatement {
818            sql: "UPDATE notes SET namespace=?1 \
819                  WHERE id=?2 AND namespace=?3 AND kind='memory' AND deleted_at IS NULL"
820                .into(),
821            params: vec![
822                SqlValue::Text(to_namespace.into()),
823                SqlValue::Text(source_id.to_string()),
824                SqlValue::Text(from_namespace.into()),
825            ],
826            label: Some("test-move-memory-source-between-check-and-restore".into()),
827        };
828        execute_private_write(runtime, statement).await
829    }
830
831    async fn soft_delete_memory_source(runtime: &KhiveRuntime, source_id: Uuid) -> u64 {
832        let statement = SqlStatement {
833            sql: "UPDATE notes SET deleted_at=?1 \
834                  WHERE id=?2 AND namespace=?3 AND kind='memory' AND deleted_at IS NULL"
835                .into(),
836            params: vec![
837                SqlValue::Integer(chrono::Utc::now().timestamp_micros()),
838                SqlValue::Text(source_id.to_string()),
839                SqlValue::Text("local".into()),
840            ],
841            label: Some("test-soft-delete-memory-source-between-check-and-restore".into()),
842        };
843        execute_private_write(runtime, statement).await
844    }
845
846    async fn change_note_kind(
847        runtime: &KhiveRuntime,
848        note_id: Uuid,
849        from_kind: &str,
850        to_kind: &str,
851    ) -> u64 {
852        let statement = SqlStatement {
853            sql: "UPDATE notes SET kind=?1 WHERE id=?2 AND kind=?3 AND deleted_at IS NULL".into(),
854            params: vec![
855                SqlValue::Text(to_kind.into()),
856                SqlValue::Text(note_id.to_string()),
857                SqlValue::Text(from_kind.into()),
858            ],
859            label: Some("test-promote-target-to-memory-between-check-and-restore".into()),
860        };
861        execute_private_write(runtime, statement).await
862    }
863
864    async fn assert_memory_note_in_namespace(
865        runtime: &KhiveRuntime,
866        token: &crate::NamespaceToken,
867        id: Uuid,
868        expected_namespace: &str,
869    ) {
870        match runtime
871            .resolve_by_id_including_deleted(token, id)
872            .await
873            .unwrap()
874        {
875            Some(Resolved::Note(note)) => {
876                assert_eq!(note.namespace, expected_namespace);
877                assert_eq!(note.kind, "memory");
878                assert!(note.deleted_at.is_none());
879            }
880            _ => panic!("fixture endpoint was not a live memory note"),
881        }
882    }
883
884    fn assert_helper_namespace_error(message: &str) {
885        assert_eq!(
886            message,
887            "restore: memory edge source namespace other does not match stored edge namespace local; leave this tombstone deleted and create a replacement link in namespace=other",
888            "restore_refusal_names_operation_and_supported_remedy"
889        );
890    }
891
892    #[tokio::test]
893    async fn restore_edge_rejects_legacy_tombstone_with_mismatched_memory_source_namespace() {
894        let (runtime, _dir) = fixture();
895        // Keep the source namespace readable for this diagnostic assertion.
896        let local_token = runtime
897            .authorize_with_visibility(Namespace::local(), vec![Namespace::parse("other").unwrap()])
898            .unwrap();
899        let other_token = runtime
900            .authorize(Namespace::parse("other").unwrap())
901            .unwrap();
902
903        let source = Note::new("other", "memory", "restore source");
904        let source_id = source.id;
905        let target = Note::new("other", "memory", "restore target");
906        let target_id = target.id;
907        runtime
908            .notes(&other_token)
909            .unwrap()
910            .upsert_note(source)
911            .await
912            .unwrap();
913        runtime
914            .notes(&other_token)
915            .unwrap()
916            .upsert_note(target)
917            .await
918            .unwrap();
919
920        let edge = runtime
921            .link(
922                &other_token,
923                source_id,
924                target_id,
925                EdgeRelation::Annotates,
926                1.0,
927                None,
928            )
929            .await
930            .unwrap();
931        let edge_id = Uuid::from(edge.id);
932        assert_eq!(edge.namespace, "other");
933        assert_eq!(edge.source_id, source_id);
934        assert_eq!(edge.target_id, target_id);
935        assert_eq!(edge.relation, EdgeRelation::Annotates);
936
937        assert_eq!(
938            prepare_tombstone(&runtime, edge_id, "other", "local").await,
939            1
940        );
941        let stored_edge = runtime
942            .get_edge_including_deleted(&local_token, edge_id)
943            .await
944            .unwrap()
945            .expect("synthetic legacy edge row should remain addressable");
946        assert_eq!(stored_edge.namespace, "local");
947        assert_eq!(stored_edge.source_id, source_id);
948        assert_eq!(stored_edge.target_id, target_id);
949        assert_eq!(stored_edge.relation, EdgeRelation::Annotates);
950        assert!(stored_edge.deleted_at.is_some());
951        assert_memory_note_in_namespace(&runtime, &local_token, source_id, "other").await;
952        assert_memory_note_in_namespace(&runtime, &local_token, target_id, "other").await;
953
954        let (entered_tx, mut entered_rx) = oneshot::channel();
955        install_before_writer_hook(edge_id, entered_tx, None);
956        let outcome = runtime.restore_edge(&local_token, edge_id).await;
957        remove_before_writer_hook(edge_id);
958        let writer_boundary_reached = matches!(entered_rx.try_recv(), Ok(()));
959        match outcome {
960            Err(RuntimeError::InvalidInput(message)) => {
961                if writer_boundary_reached {
962                    panic!("restore_memory_namespace_guard_bypassed_writer_reached");
963                }
964                assert_helper_namespace_error(&message);
965            }
966            Ok(Some((_, true))) => panic!("restore_memory_namespace_mismatch_admitted"),
967            _ => panic!("fixture did not reach the targeted restore refusal"),
968        }
969
970        let after_refusal = runtime
971            .get_edge_including_deleted(&local_token, edge_id)
972            .await
973            .unwrap()
974            .expect("refused restore must preserve the tombstone row");
975        assert_eq!(after_refusal.namespace, "local");
976        assert!(after_refusal.deleted_at.is_some());
977    }
978
979    #[tokio::test]
980    async fn restore_edge_rechecks_memory_source_namespace_at_writer_after_precheck() {
981        let (runtime, _dir) = fixture();
982        let local_token = runtime.authorize(Namespace::local()).unwrap();
983        let other_token = runtime
984            .authorize(Namespace::parse("other").unwrap())
985            .unwrap();
986
987        let source = Note::new("local", "memory", "moving restore source");
988        let source_id = source.id;
989        let target = Note::new("local", "memory", "stable restore target");
990        let target_id = target.id;
991        runtime
992            .notes(&local_token)
993            .unwrap()
994            .upsert_note(source)
995            .await
996            .unwrap();
997        runtime
998            .notes(&local_token)
999            .unwrap()
1000            .upsert_note(target)
1001            .await
1002            .unwrap();
1003
1004        let edge = runtime
1005            .link(
1006                &local_token,
1007                source_id,
1008                target_id,
1009                EdgeRelation::Annotates,
1010                1.0,
1011                None,
1012            )
1013            .await
1014            .unwrap();
1015        let edge_id = Uuid::from(edge.id);
1016        assert_eq!(edge.namespace, "local");
1017        assert_eq!(edge.source_id, source_id);
1018        assert_eq!(edge.target_id, target_id);
1019        assert_eq!(edge.relation, EdgeRelation::Annotates);
1020        assert_eq!(
1021            prepare_tombstone(&runtime, edge_id, "local", "local").await,
1022            1
1023        );
1024        let edge_before_restore = runtime
1025            .get_edge_including_deleted(&local_token, edge_id)
1026            .await
1027            .unwrap()
1028            .expect("tombstone should be present before restore");
1029        assert_eq!(edge_before_restore.namespace, "local");
1030        assert!(edge_before_restore.deleted_at.is_some());
1031        assert_memory_note_in_namespace(&runtime, &local_token, source_id, "local").await;
1032        assert_memory_note_in_namespace(&runtime, &local_token, target_id, "local").await;
1033
1034        let (entered_tx, mut entered_rx) = oneshot::channel();
1035        let (release_tx, release_rx) = oneshot::channel();
1036        install_before_writer_hook(edge_id, entered_tx, Some(release_rx));
1037        let mut restore = Box::pin(runtime.restore_edge(&local_token, edge_id));
1038        let crossed_precheck = tokio::time::timeout(Duration::from_secs(10), async {
1039            tokio::select! {
1040                _ = &mut restore => false,
1041                reached = &mut entered_rx => reached.is_ok(),
1042            }
1043        })
1044        .await
1045        .unwrap_or(false);
1046        if !crossed_precheck {
1047            remove_before_writer_hook(edge_id);
1048            panic!("fixture did not positively cross the outside guard before writer");
1049        }
1050
1051        assert_eq!(
1052            move_memory_source_namespace(&runtime, source_id, "local", "other").await,
1053            1
1054        );
1055        assert_memory_note_in_namespace(&runtime, &other_token, source_id, "other").await;
1056        assert_memory_note_in_namespace(&runtime, &local_token, target_id, "local").await;
1057        let edge_while_paused = runtime
1058            .get_edge_including_deleted(&local_token, edge_id)
1059            .await
1060            .unwrap()
1061            .expect("edge must still exist while restore is held before writer");
1062        assert_eq!(edge_while_paused.namespace, "local");
1063        assert!(edge_while_paused.deleted_at.is_some());
1064
1065        assert!(release_tx.send(()).is_ok());
1066        let outcome = tokio::time::timeout(Duration::from_secs(10), &mut restore)
1067            .await
1068            .expect("restore did not complete after release");
1069        match outcome {
1070            Err(RuntimeError::InvalidInput(_)) => {}
1071            Ok(Some((_, true))) => {
1072                panic!("restore_memory_namespace_drift_admitted_after_precheck")
1073            }
1074            _ => panic!("writer-fence fixture did not return the expected refusal class"),
1075        }
1076
1077        let after_refusal = runtime
1078            .get_edge_including_deleted(&local_token, edge_id)
1079            .await
1080            .unwrap()
1081            .expect("writer-fence refusal must preserve the tombstone");
1082        assert_eq!(after_refusal.namespace, "local");
1083        assert!(after_refusal.deleted_at.is_some());
1084    }
1085
1086    #[tokio::test]
1087    async fn restore_edge_rechecks_memory_source_liveness_at_writer_after_precheck() {
1088        let (runtime, _dir) = fixture();
1089        let local_token = runtime.authorize(Namespace::local()).unwrap();
1090
1091        let source = Note::new("local", "memory", "soft-deleted restore source");
1092        let source_id = source.id;
1093        let target = Note::new("local", "memory", "live restore target");
1094        let target_id = target.id;
1095        runtime
1096            .notes(&local_token)
1097            .unwrap()
1098            .upsert_note(source)
1099            .await
1100            .unwrap();
1101        runtime
1102            .notes(&local_token)
1103            .unwrap()
1104            .upsert_note(target)
1105            .await
1106            .unwrap();
1107
1108        let edge = runtime
1109            .link(
1110                &local_token,
1111                source_id,
1112                target_id,
1113                EdgeRelation::Annotates,
1114                1.0,
1115                None,
1116            )
1117            .await
1118            .unwrap();
1119        let edge_id = Uuid::from(edge.id);
1120        assert_eq!(edge.namespace, "local");
1121        assert_eq!(
1122            prepare_tombstone(&runtime, edge_id, "local", "local").await,
1123            1
1124        );
1125        assert_memory_note_in_namespace(&runtime, &local_token, source_id, "local").await;
1126        assert_memory_note_in_namespace(&runtime, &local_token, target_id, "local").await;
1127
1128        let (entered_tx, mut entered_rx) = oneshot::channel();
1129        let (release_tx, release_rx) = oneshot::channel();
1130        install_before_writer_hook(edge_id, entered_tx, Some(release_rx));
1131        let mut restore = Box::pin(runtime.restore_edge(&local_token, edge_id));
1132        let crossed_precheck = tokio::time::timeout(Duration::from_secs(10), async {
1133            tokio::select! {
1134                _ = &mut restore => false,
1135                reached = &mut entered_rx => reached.is_ok(),
1136            }
1137        })
1138        .await
1139        .unwrap_or(false);
1140        if !crossed_precheck {
1141            remove_before_writer_hook(edge_id);
1142            panic!("fixture did not positively cross the outside guard before writer");
1143        }
1144
1145        assert_eq!(soft_delete_memory_source(&runtime, source_id).await, 1);
1146        match runtime
1147            .resolve_by_id_including_deleted(&local_token, source_id)
1148            .await
1149            .unwrap()
1150        {
1151            Some(Resolved::Note(note)) => {
1152                assert_eq!(note.namespace, "local");
1153                assert_eq!(note.kind, "memory");
1154                assert!(note.deleted_at.is_some());
1155            }
1156            _ => panic!("fixture source did not remain as an included-deleted memory note"),
1157        }
1158        assert_memory_note_in_namespace(&runtime, &local_token, target_id, "local").await;
1159
1160        assert!(release_tx.send(()).is_ok());
1161        let outcome = tokio::time::timeout(Duration::from_secs(10), &mut restore)
1162            .await
1163            .expect("restore did not complete after release");
1164        match outcome {
1165            Err(RuntimeError::InvalidInput(_)) => {}
1166            Ok(Some((_, true))) => {
1167                panic!("restore_memory_source_soft_delete_admitted_after_precheck")
1168            }
1169            _ => panic!("source-liveness fence fixture did not return the expected refusal class"),
1170        }
1171        let after_refusal = runtime
1172            .get_edge_including_deleted(&local_token, edge_id)
1173            .await
1174            .unwrap()
1175            .expect("source-liveness refusal must preserve the tombstone");
1176        assert_eq!(after_refusal.namespace, "local");
1177        assert!(after_refusal.deleted_at.is_some());
1178    }
1179
1180    #[tokio::test]
1181    async fn restore_edge_rejects_preexisting_soft_deleted_memory_source() {
1182        let (runtime, _dir) = fixture();
1183        let local_token = runtime.authorize(Namespace::local()).unwrap();
1184        let source = Note::new("local", "memory", "already deleted restore source");
1185        let source_id = source.id;
1186        let target = Note::new("local", "memory", "live restore target");
1187        let target_id = target.id;
1188        runtime
1189            .notes(&local_token)
1190            .unwrap()
1191            .upsert_note(source)
1192            .await
1193            .unwrap();
1194        runtime
1195            .notes(&local_token)
1196            .unwrap()
1197            .upsert_note(target)
1198            .await
1199            .unwrap();
1200
1201        let edge = runtime
1202            .link(
1203                &local_token,
1204                source_id,
1205                target_id,
1206                EdgeRelation::Annotates,
1207                1.0,
1208                None,
1209            )
1210            .await
1211            .unwrap();
1212        let edge_id = Uuid::from(edge.id);
1213        assert_eq!(
1214            prepare_tombstone(&runtime, edge_id, "local", "local").await,
1215            1
1216        );
1217        let stored_edge = runtime
1218            .get_edge_including_deleted(&local_token, edge_id)
1219            .await
1220            .unwrap()
1221            .expect("fixture tombstone should be addressable before source deletion");
1222        assert_eq!(stored_edge.namespace, "local");
1223        assert!(stored_edge.deleted_at.is_some());
1224        assert_eq!(soft_delete_memory_source(&runtime, source_id).await, 1);
1225        match runtime
1226            .resolve_by_id_including_deleted(&local_token, source_id)
1227            .await
1228            .unwrap()
1229        {
1230            Some(Resolved::Note(note)) => {
1231                assert_eq!(note.namespace, "local");
1232                assert_eq!(note.kind, "memory");
1233                assert!(note.deleted_at.is_some());
1234            }
1235            _ => panic!("fixture source did not remain as an included-deleted memory note"),
1236        }
1237        assert_memory_note_in_namespace(&runtime, &local_token, target_id, "local").await;
1238
1239        let (entered_tx, mut entered_rx) = oneshot::channel();
1240        install_before_writer_hook(edge_id, entered_tx, None);
1241        let outcome = runtime.restore_edge(&local_token, edge_id).await;
1242        remove_before_writer_hook(edge_id);
1243        assert!(
1244            matches!(entered_rx.try_recv(), Ok(())),
1245            "fixture did not reach writer fence"
1246        );
1247        match outcome {
1248            Err(RuntimeError::InvalidInput(_)) => {}
1249            Ok(Some((_, true))) => panic!("restore_memory_predeleted_source_admitted"),
1250            _ => panic!("predeleted-source fixture did not return the expected refusal class"),
1251        }
1252        let after_refusal = runtime
1253            .get_edge_including_deleted(&local_token, edge_id)
1254            .await
1255            .unwrap()
1256            .expect("predeleted-source refusal must preserve the tombstone");
1257        assert!(after_refusal.deleted_at.is_some());
1258    }
1259
1260    #[tokio::test]
1261    async fn restore_edge_rechecks_current_memory_pair_after_nonmemory_precheck() {
1262        let (runtime, _dir) = fixture();
1263        let local_token = runtime.authorize(Namespace::local()).unwrap();
1264        let other_token = runtime
1265            .authorize(Namespace::parse("other").unwrap())
1266            .unwrap();
1267        let source = Note::new(
1268            "other",
1269            "memory",
1270            "memory source before current-pair change",
1271        );
1272        let source_id = source.id;
1273        let target = Note::new(
1274            "other",
1275            "observation",
1276            "nonmemory target before current-pair change",
1277        );
1278        let target_id = target.id;
1279        runtime
1280            .notes(&other_token)
1281            .unwrap()
1282            .upsert_note(source)
1283            .await
1284            .unwrap();
1285        runtime
1286            .notes(&other_token)
1287            .unwrap()
1288            .upsert_note(target)
1289            .await
1290            .unwrap();
1291
1292        let edge = runtime
1293            .link(
1294                &other_token,
1295                source_id,
1296                target_id,
1297                EdgeRelation::Annotates,
1298                1.0,
1299                None,
1300            )
1301            .await
1302            .unwrap();
1303        let edge_id = Uuid::from(edge.id);
1304        assert_eq!(
1305            prepare_tombstone(&runtime, edge_id, "other", "local").await,
1306            1
1307        );
1308        let edge_before_restore = runtime
1309            .get_edge_including_deleted(&local_token, edge_id)
1310            .await
1311            .unwrap()
1312            .expect("current-pair fixture tombstone should be addressable");
1313        assert_eq!(edge_before_restore.namespace, "local");
1314        assert_eq!(edge_before_restore.source_id, source_id);
1315        assert_eq!(edge_before_restore.target_id, target_id);
1316        assert!(edge_before_restore.deleted_at.is_some());
1317        assert_memory_note_in_namespace(&runtime, &local_token, source_id, "other").await;
1318        match runtime
1319            .resolve_by_id_including_deleted(&local_token, target_id)
1320            .await
1321            .unwrap()
1322        {
1323            Some(Resolved::Note(note)) => {
1324                assert_eq!(note.namespace, "other");
1325                assert_eq!(note.kind, "observation");
1326                assert!(note.deleted_at.is_none());
1327            }
1328            _ => panic!("fixture target was not the expected nonmemory Note"),
1329        }
1330
1331        match runtime.restore_edge(&local_token, edge_id).await {
1332            Ok(Some((restored, true))) => {
1333                assert_eq!(restored.namespace, "local");
1334                assert!(restored.deleted_at.is_none());
1335            }
1336            _ => panic!("ordinary memory-to-observation restore should preserve existing behavior"),
1337        }
1338        let ordinary_restore = runtime
1339            .get_edge_including_deleted(&local_token, edge_id)
1340            .await
1341            .unwrap()
1342            .expect("ordinary nonmemory-pair restore should remain readable");
1343        assert_eq!(ordinary_restore.namespace, "local");
1344        assert!(ordinary_restore.deleted_at.is_none());
1345        assert_eq!(
1346            prepare_tombstone(&runtime, edge_id, "local", "local").await,
1347            1
1348        );
1349        let edge_before_race = runtime
1350            .get_edge_including_deleted(&local_token, edge_id)
1351            .await
1352            .unwrap()
1353            .expect("retombstoned edge should be addressable before current-pair race");
1354        assert_eq!(edge_before_race.namespace, "local");
1355        assert!(edge_before_race.deleted_at.is_some());
1356
1357        let (entered_tx, mut entered_rx) = oneshot::channel();
1358        let (release_tx, release_rx) = oneshot::channel();
1359        install_before_writer_hook(edge_id, entered_tx, Some(release_rx));
1360        let mut restore = Box::pin(runtime.restore_edge(&local_token, edge_id));
1361        let crossed_precheck = tokio::time::timeout(Duration::from_secs(10), async {
1362            tokio::select! {
1363                _ = &mut restore => false,
1364                reached = &mut entered_rx => reached.is_ok(),
1365            }
1366        })
1367        .await
1368        .unwrap_or(false);
1369        if !crossed_precheck {
1370            remove_before_writer_hook(edge_id);
1371            panic!("fixture did not positively cross the outside check before writer");
1372        }
1373
1374        assert_eq!(
1375            change_note_kind(&runtime, target_id, "observation", "memory").await,
1376            1
1377        );
1378        match runtime
1379            .resolve_by_id_including_deleted(&local_token, target_id)
1380            .await
1381            .unwrap()
1382        {
1383            Some(Resolved::Note(note)) => {
1384                assert_eq!(note.namespace, "other");
1385                assert_eq!(note.kind, "memory");
1386                assert!(note.deleted_at.is_none());
1387            }
1388            _ => panic!("fixture target did not become a live memory Note"),
1389        }
1390        assert!(release_tx.send(()).is_ok());
1391        let outcome = tokio::time::timeout(Duration::from_secs(10), &mut restore)
1392            .await
1393            .expect("restore did not complete after release");
1394        match outcome {
1395            Err(RuntimeError::InvalidInput(_)) => {}
1396            Ok(Some((_, true))) => {
1397                panic!("restore_current_memory_pair_source_namespace_admitted")
1398            }
1399            _ => panic!("current-pair fixture did not return the expected refusal class"),
1400        }
1401        let after_refusal = runtime
1402            .get_edge_including_deleted(&local_token, edge_id)
1403            .await
1404            .unwrap()
1405            .expect("current-pair refusal must preserve the tombstone");
1406        assert_eq!(after_refusal.namespace, "local");
1407        assert!(after_refusal.deleted_at.is_some());
1408    }
1409
1410    #[tokio::test]
1411    async fn restore_edge_allows_memory_target_in_another_namespace() {
1412        let (runtime, _dir) = fixture();
1413        let local_token = runtime.authorize(Namespace::local()).unwrap();
1414        let other_token = runtime
1415            .authorize(Namespace::parse("other").unwrap())
1416            .unwrap();
1417
1418        let source = Note::new("local", "memory", "local restore source");
1419        let source_id = source.id;
1420        let target = Note::new("other", "memory", "foreign restore target");
1421        let target_id = target.id;
1422        runtime
1423            .notes(&local_token)
1424            .unwrap()
1425            .upsert_note(source)
1426            .await
1427            .unwrap();
1428        runtime
1429            .notes(&other_token)
1430            .unwrap()
1431            .upsert_note(target)
1432            .await
1433            .unwrap();
1434
1435        let edge = runtime
1436            .link(
1437                &local_token,
1438                source_id,
1439                target_id,
1440                EdgeRelation::Annotates,
1441                1.0,
1442                None,
1443            )
1444            .await
1445            .unwrap();
1446        let edge_id = Uuid::from(edge.id);
1447        assert_eq!(edge.namespace, "local");
1448        assert_eq!(edge.source_id, source_id);
1449        assert_eq!(edge.target_id, target_id);
1450        assert_eq!(
1451            prepare_tombstone(&runtime, edge_id, "local", "local").await,
1452            1
1453        );
1454        assert_memory_note_in_namespace(&runtime, &local_token, source_id, "local").await;
1455        assert_memory_note_in_namespace(&runtime, &local_token, target_id, "other").await;
1456
1457        match runtime.restore_edge(&local_token, edge_id).await {
1458            Ok(Some((restored, true))) => {
1459                assert_eq!(restored.namespace, "local");
1460                assert!(restored.deleted_at.is_none());
1461            }
1462            _ => panic!("source-local memory edge with a cross-namespace target should restore"),
1463        }
1464        let after_restore = runtime
1465            .get_edge_including_deleted(&local_token, edge_id)
1466            .await
1467            .unwrap()
1468            .expect("restored edge should remain readable");
1469        assert_eq!(after_restore.namespace, "local");
1470        assert!(after_restore.deleted_at.is_none());
1471    }
1472
1473    #[tokio::test]
1474    async fn restore_refusal_hides_unreadable_memory_source_namespace() {
1475        let (runtime, _dir) = fixture();
1476        let source_namespace = "guard-private-source";
1477        let source_token = runtime
1478            .authorize(Namespace::parse(source_namespace).unwrap())
1479            .unwrap();
1480        let readable = runtime
1481            .authorize_with_visibility(
1482                Namespace::local(),
1483                vec![Namespace::parse(source_namespace).unwrap()],
1484            )
1485            .unwrap();
1486        let unreadable = runtime
1487            .authorize_with_visibility(Namespace::local(), Vec::new())
1488            .unwrap();
1489        assert!(readable
1490            .visible_namespaces()
1491            .iter()
1492            .any(|ns| ns.as_str() == source_namespace));
1493        assert!(!unreadable
1494            .visible_namespaces()
1495            .iter()
1496            .any(|ns| ns.as_str() == source_namespace));
1497        let source = Note::new(source_namespace, "memory", "source");
1498        let target = Note::new(source_namespace, "memory", "target");
1499        let (source_id, target_id) = (source.id, target.id);
1500        runtime
1501            .notes(&source_token)
1502            .unwrap()
1503            .upsert_note(source)
1504            .await
1505            .unwrap();
1506        runtime
1507            .notes(&source_token)
1508            .unwrap()
1509            .upsert_note(target)
1510            .await
1511            .unwrap();
1512        assert_memory_note_in_namespace(&runtime, &source_token, source_id, source_namespace).await;
1513        assert_memory_note_in_namespace(&runtime, &source_token, target_id, source_namespace).await;
1514        let edge = runtime
1515            .link(
1516                &source_token,
1517                source_id,
1518                target_id,
1519                EdgeRelation::Annotates,
1520                1.0,
1521                None,
1522            )
1523            .await
1524            .expect("matching source namespace must admit the fixture link");
1525        assert_eq!(edge.namespace.as_str(), source_namespace);
1526        let edge_id = Uuid::from(edge.id);
1527        assert_eq!(
1528            prepare_tombstone(&runtime, edge_id, source_namespace, "local").await,
1529            1
1530        );
1531        let before = runtime
1532            .get_edge_including_deleted(&unreadable, edge_id)
1533            .await
1534            .unwrap()
1535            .unwrap();
1536        assert_eq!(before.namespace.as_str(), "local");
1537        assert!(before.deleted_at.is_some());
1538        let RuntimeError::InvalidInput(message) =
1539            runtime.restore_edge(&readable, edge_id).await.unwrap_err()
1540        else {
1541            panic!("readable mismatch must preserve InvalidInput");
1542        };
1543        assert_eq!(message, format!("restore: memory edge source namespace {source_namespace} does not match stored edge namespace local; leave this tombstone deleted and create a replacement link in namespace={source_namespace}"));
1544        let RuntimeError::InvalidInput(message) = runtime
1545            .restore_edge(&unreadable, edge_id)
1546            .await
1547            .unwrap_err()
1548        else {
1549            panic!("unreadable mismatch must preserve InvalidInput");
1550        };
1551        assert!(
1552            !message.contains(source_namespace),
1553            "restore_unreadable_source_namespace_not_disclosed"
1554        );
1555        assert!(!message.contains(&source_id.to_string()));
1556        assert!(!message.contains(&target_id.to_string()));
1557        assert_eq!(message, "restore: memory edge source row is in a namespace the caller cannot read; the edge cannot be restored under stored edge namespace local");
1558        let after = runtime
1559            .get_edge_including_deleted(&unreadable, edge_id)
1560            .await
1561            .unwrap()
1562            .unwrap();
1563        assert_eq!(after.namespace.as_str(), "local");
1564        assert_eq!(after.source_id, source_id);
1565        assert_eq!(after.target_id, target_id);
1566        assert!(after.deleted_at.is_some());
1567    }
1568}