Skip to main content

khive_runtime/operations/
delete_restore.rs

1use chrono::Utc;
2use uuid::Uuid;
3
4use khive_db::stores::entity::entity_hard_delete_statement;
5use khive_db::stores::event::hard_delete_lineage_warning_statements;
6use khive_db::stores::graph::purge_incident_edges_statement;
7use khive_db::stores::note::note_hard_delete_statement;
8use khive_db::stores::text::insert_document_statements;
9use khive_storage::types::{DeleteMode, SqlStatement, SqlValue};
10use khive_storage::{AttachmentSubstrate, Edge, Entity};
11use khive_types::{EventKind, SubstrateKind};
12
13use crate::atomic_plan::{
14    AffectedRowGuard, DeletePlan, PlanStatement, PostCommitEffect, UpdatePlan,
15};
16use crate::atomic_runner::{run_atomic_unit, AtomicOpFailure, AtomicOpPlan, AtomicRunOutcome};
17use crate::curation::entity_fts_document;
18use crate::error::{RuntimeError, RuntimeResult};
19use crate::runtime::{KhiveRuntime, NamespaceToken};
20
21#[cfg(any(test, feature = "fault-injection"))]
22use super::{consume_fault, FTS_FAIL_NS, ROLLBACK_CLEANUP_FAIL_NS};
23use super::{
24    legacy_post_commit_result, live_merged_entity_refused, merge_tombstone_restore_refused,
25    record_post_commit_degradation, restore_reindex_failed, PostCommitDegradation,
26};
27
28impl KhiveRuntime {
29    /// Hard-delete a single graph node (entity, note, or edge-as-node row) AND purge its
30    /// incident edges in ONE write transaction — closes a race where a concurrent guarded
31    /// write could insert a fresh edge against the endpoint between two separately
32    /// committed calls; see docs/operations.md#atomic_hard_delete_with_edge_purge.
33    ///
34    /// `row_statement` is the exact hard-delete `DELETE` for the target row
35    /// (entity, note, or edge). Before the incident-edge purge, the same plan
36    /// appends any ADR-002 lineage-loss warnings from the still-present edge
37    /// rows, so the warning payload and cascade commit or roll back together.
38    /// Returns `Ok(true)` if the row was deleted, `Ok(false)` if it no longer
39    /// existed (lost a race with a concurrent delete of the same row).
40    pub(super) async fn atomic_hard_delete_with_edge_purge(
41        &self,
42        row_statement: SqlStatement,
43        node_id: Uuid,
44        namespace: &str,
45        actor: &str,
46        substrate: SubstrateKind,
47    ) -> RuntimeResult<bool> {
48        let mut statements = vec![PlanStatement {
49            statement: row_statement,
50            guard: Some(AffectedRowGuard::exactly(1)),
51        }];
52        if matches!(substrate, SubstrateKind::Entity | SubstrateKind::Note) {
53            statements.push(PlanStatement {
54                statement: khive_db::stores::attachment::delete_record_attachments_statement(
55                    node_id,
56                    if substrate == SubstrateKind::Entity {
57                        AttachmentSubstrate::Entity
58                    } else {
59                        AttachmentSubstrate::Note
60                    },
61                ),
62                guard: None,
63            });
64        }
65        statements.extend(
66            hard_delete_lineage_warning_statements(namespace, actor, node_id, substrate)
67                .into_iter()
68                .map(|statement| PlanStatement {
69                    statement,
70                    guard: None,
71                }),
72        );
73        statements.push(PlanStatement {
74            statement: purge_incident_edges_statement(node_id),
75            guard: None,
76        });
77        let plan = AtomicOpPlan::Delete(DeletePlan {
78            target_id: node_id,
79            statements,
80            post_commit: PostCommitEffect::None,
81        });
82        match run_atomic_unit(self.sql().as_ref(), vec![plan]).await {
83            Ok(AtomicRunOutcome::Committed { .. }) => Ok(true),
84            Ok(AtomicRunOutcome::RolledBack {
85                failure: AtomicOpFailure::NoteConflict(conflict),
86                ..
87            }) => Err(conflict.into_error().into()),
88            Ok(AtomicRunOutcome::RolledBack {
89                failure: AtomicOpFailure::EntityConflict(conflict),
90                ..
91            }) => Err(conflict.into_error().into()),
92            Ok(AtomicRunOutcome::RolledBack {
93                failure: AtomicOpFailure::GuardFailed { .. },
94                ..
95            }) => Ok(false),
96            Ok(AtomicRunOutcome::RolledBack {
97                failure: AtomicOpFailure::SqlError { message, .. },
98                ..
99            }) => Err(RuntimeError::Internal(format!(
100                "hard delete + edge purge for {node_id} failed: {message}"
101            ))),
102            // Only canonical change-set plans compare preimages or owner snapshots.
103            Ok(AtomicRunOutcome::RolledBack {
104                failure:
105                    failure @ (AtomicOpFailure::PreimageMismatch { .. }
106                    | AtomicOpFailure::OwnerSnapshotConflict { .. }),
107                ..
108            }) => Err(RuntimeError::Internal(format!(
109                "hard delete + edge purge for {node_id}: unexpected canonical failure {failure:?}"
110            ))),
111            Err(e) => Err(RuntimeError::Internal(format!(
112                "hard delete + edge purge for {node_id}: atomic unit seam failure: {}",
113                e.0
114            ))),
115        }
116    }
117
118    /// Restore an entity tombstone owned by the caller's primary namespace.
119    ///
120    /// The restore is guarded by both the tombstone's id/namespace and the
121    /// current uniqueness state, so a caller cannot resurrect over a newer
122    /// live record. Indexes are rebuilt only after the row restore commits.
123    pub async fn restore_entity(
124        &self,
125        token: &NamespaceToken,
126        id: Uuid,
127    ) -> RuntimeResult<Option<(Entity, bool)>> {
128        let Some(entity) = self
129            .entities(token)?
130            .get_entity_including_deleted(id)
131            .await?
132        else {
133            return Ok(None);
134        };
135        if entity.namespace != token.namespace().as_str() {
136            return Ok(None);
137        }
138        // A merge tombstone is not a plain soft delete: the source row carries
139        // merge provenance and its content already lives on the kept entity.
140        // Clearing only `deleted_at` would bring the source back as a live
141        // duplicate that still claims to have been merged. Refuse and name
142        // the kept id; restore does not undo a merge.
143        //
144        // The merge check runs before the already-live short cut on purpose:
145        // a live row that still carries `merged_into` is what an earlier
146        // restore left behind before this guard existed, and answering it
147        // "already live" would hide the invariant violation from the one
148        // caller who is looking at the row. Name it instead.
149        if let Some(kept_id) = entity.merged_into {
150            if entity.deleted_at.is_none() {
151                return Err(live_merged_entity_refused(id, kept_id));
152            }
153            return Err(merge_tombstone_restore_refused(id, kept_id));
154        }
155        if entity.deleted_at.is_none() {
156            return Ok(Some((entity, false)));
157        }
158        let updated_at =
159            Utc::now()
160                .timestamp_micros()
161                .max(entity.updated_at.checked_add(1).ok_or_else(|| {
162                    RuntimeError::Internal(format!(
163                        "entity {id} updated_at is already at i64::MAX and cannot advance"
164                    ))
165                })?);
166        let mut restored = entity;
167        restored.deleted_at = None;
168        restored.updated_at = updated_at;
169        restored.version = restored
170            .version
171            .checked_add(1)
172            .ok_or_else(|| RuntimeError::InvalidInput("entity version overflow".into()))?;
173        let mut statements = vec![PlanStatement {
174            statement: SqlStatement {
175                sql: "UPDATE entities SET deleted_at=NULL, updated_at=?1, version=version+1 \
176                      WHERE id=?2 AND namespace=?3 AND deleted_at IS NOT NULL AND version=?4"
177                    .into(),
178                params: vec![
179                    SqlValue::Integer(updated_at),
180                    SqlValue::Text(id.to_string()),
181                    SqlValue::Text(token.namespace().as_str().to_owned()),
182                    SqlValue::Integer(restored.version - 1),
183                ],
184                label: Some("entity-restore".into()),
185            },
186            guard: Some(AffectedRowGuard::exactly(1)),
187        }];
188        // The soft delete removed the FTS row, so the text index is published
189        // in the same unit as the row: a live row that search cannot find is
190        // not a state this verb can leave behind. Order-sensitive pair — see
191        // `insert_document_statements`'s adjacency contract.
192        for statement in khive_db::stores::text::delete_document_statements(
193            "fts_entities",
194            &restored.namespace,
195            id,
196        )
197        .into_iter()
198        .chain(insert_document_statements(
199            "fts_entities",
200            &entity_fts_document(&restored),
201        )) {
202            statements.push(PlanStatement {
203                statement,
204                guard: None,
205            });
206        }
207        let plan = AtomicOpPlan::Update(Box::new(UpdatePlan {
208            graph_effects: Vec::new(),
209            target_id: id,
210            statements,
211            post_commit: PostCommitEffect::None,
212            edge_natural_key: None,
213            idempotent_noop: false,
214            entity_guard: None,
215            note_guard: None,
216            note_vector_purge: None,
217            note_embedding_inheritance: None,
218        }));
219        match run_atomic_unit(self.sql().as_ref(), vec![plan]).await {
220            Ok(AtomicRunOutcome::Committed { .. }) => {
221                // Embeddings are rebuilt after the commit; the row and its
222                // text index are already live, so a failure here names that.
223                #[cfg(any(test, feature = "fault-injection"))]
224                if consume_fault(&FTS_FAIL_NS, &restored.namespace) {
225                    return Err(restore_reindex_failed(
226                        "entity",
227                        id,
228                        RuntimeError::Internal("injected FTS failure".to_string()),
229                    ));
230                }
231                self.reindex_entity(token, &restored)
232                    .await
233                    .map_err(|e| restore_reindex_failed("entity", id, e))?;
234                Ok(Some((restored, true)))
235            }
236            Ok(AtomicRunOutcome::RolledBack { failure, .. }) => Err(RuntimeError::Internal(
237                format!("entity restore rolled back: {failure:?}"),
238            )),
239            Err(error) => Err(RuntimeError::Storage(error.0)),
240        }
241    }
242
243    /// Restore an edge tombstone owned by the caller's primary namespace.
244    pub async fn restore_edge(
245        &self,
246        token: &NamespaceToken,
247        id: Uuid,
248    ) -> RuntimeResult<Option<(Edge, bool)>> {
249        let Some(edge) = self.get_edge_including_deleted(token, id).await? else {
250            return Ok(None);
251        };
252        if edge.namespace != token.namespace().as_str() {
253            return Ok(None);
254        }
255        if edge.deleted_at.is_none() {
256            return Ok(Some((edge, false)));
257        }
258        let both_memory_at_check = match (
259            self.resolve_by_id_including_deleted(token, edge.source_id)
260                .await?,
261            self.resolve_by_id_including_deleted(token, edge.target_id)
262                .await?,
263        ) {
264            (Some(crate::Resolved::Note(source)), Some(crate::Resolved::Note(target))) => {
265                let both_memory = source.kind == "memory" && target.kind == "memory";
266                Self::ensure_memory_note_edge_namespace(
267                    token,
268                    "restore",
269                    &edge.namespace,
270                    (edge.source_id, edge.target_id),
271                    edge.relation,
272                    &source,
273                    &target,
274                )?;
275                both_memory
276            }
277            _ => false,
278        };
279        let updated_at = Utc::now();
280        let plan = AtomicOpPlan::Update(Box::new(UpdatePlan {
281            graph_effects: Vec::new(),
282            target_id: id,
283            statements: vec![PlanStatement {
284                statement: SqlStatement {
285                    sql: "UPDATE graph_edges SET deleted_at=NULL, updated_at=?1 \
286                          WHERE id=?2 AND namespace=?3 AND deleted_at IS NOT NULL \
287                            AND ((?4 = 0 AND ( \
288                                  NOT EXISTS (SELECT 1 FROM notes AS source_memory \
289                                       WHERE source_memory.id=graph_edges.source_id \
290                                         AND source_memory.kind='memory') \
291                                  OR NOT EXISTS (SELECT 1 FROM notes AS target_memory \
292                                       WHERE target_memory.id=graph_edges.target_id \
293                                         AND target_memory.kind='memory') \
294                                 )) \
295                                 OR EXISTS (SELECT 1 FROM notes AS source_match \
296                                      WHERE source_match.id=graph_edges.source_id \
297                                        AND source_match.namespace=graph_edges.namespace \
298                                        AND source_match.kind='memory' \
299                                        AND source_match.deleted_at IS NULL))"
300                        .into(),
301                    params: vec![
302                        SqlValue::Integer(updated_at.timestamp_micros()),
303                        SqlValue::Text(id.to_string()),
304                        SqlValue::Text(edge.namespace.clone()),
305                        SqlValue::Integer(if both_memory_at_check { 1 } else { 0 }),
306                    ],
307                    label: Some("edge-restore".into()),
308                },
309                guard: Some(AffectedRowGuard::exactly(1)),
310            }],
311            post_commit: PostCommitEffect::None,
312            edge_natural_key: None,
313            idempotent_noop: false,
314            entity_guard: None,
315            note_guard: None,
316            note_vector_purge: None,
317            note_embedding_inheritance: None,
318        }));
319        #[cfg(test)]
320        memory_edge_namespace_restore_tests::before_writer(id).await;
321        match run_atomic_unit(self.sql().as_ref(), vec![plan]).await {
322            Ok(AtomicRunOutcome::Committed { .. }) => {
323                let mut restored = edge;
324                restored.deleted_at = None;
325                restored.updated_at = updated_at;
326                Ok(Some((restored, true)))
327            }
328            Ok(AtomicRunOutcome::RolledBack {
329                failure:
330                    AtomicOpFailure::GuardFailed {
331                        statement_label: Some(label),
332                        expected,
333                        observed: 0,
334                    },
335                ..
336            }) if label == "edge-restore" && expected == AffectedRowGuard::exactly(1) => {
337                Err(RuntimeError::InvalidInput(
338                    "edge restore preconditions changed before commit; retry against the current edge and endpoint state".into(),
339                ))
340            }
341            Ok(AtomicRunOutcome::RolledBack { failure, .. }) => Err(RuntimeError::Internal(
342                format!("edge restore rolled back: {failure:?}"),
343            )),
344            Err(error) => Err(RuntimeError::Storage(error.0)),
345        }
346    }
347
348    /// Soft-delete or hard-delete a note by ID.
349    ///
350    /// On hard delete, cascades to remove all incident edges (both inbound and
351    /// outbound) and cleans up FTS and vector indexes, preventing dangling
352    /// references for `annotates` edges that target this note.
353    /// Soft delete also cleans FTS and vector indexes; edges are left in place.
354    ///
355    /// UUID v4 is globally unique: no namespace filter on by-ID ops.
356    /// Cascade and index cleanup target the RECORD's stored namespace, not the caller token's.
357    /// Returns `Ok(false)` if the note does not exist.
358    pub async fn delete_note(
359        &self,
360        token: &NamespaceToken,
361        id: Uuid,
362        hard: bool,
363    ) -> RuntimeResult<bool> {
364        let (deleted, degradations) = self
365            .delete_note_with_post_commit_report(token, id, hard)
366            .await?;
367        legacy_post_commit_result("delete_note", id, deleted, degradations)
368    }
369
370    /// Delete the note and return diagnostics for any failed work after the
371    /// row change committed. A degradation is non-retryable: callers must not
372    /// repeat a create or delete because an index or telemetry leg failed.
373    pub async fn delete_note_with_post_commit_report(
374        &self,
375        token: &NamespaceToken,
376        id: Uuid,
377        hard: bool,
378    ) -> RuntimeResult<(bool, Vec<PostCommitDegradation>)> {
379        let note_store = self.notes(token)?;
380        let note = if hard {
381            match note_store.get_note_including_deleted(id).await? {
382                Some(n) => n,
383                None => return Ok((false, Vec::new())),
384            }
385        } else {
386            match note_store.get_note(id).await? {
387                Some(n) => n,
388                None => return Ok((false, Vec::new())),
389            }
390        };
391        if let Some(error) = self.stream_member_error(&note).await? {
392            return Err(error);
393        }
394        let mode = if hard {
395            DeleteMode::Hard
396        } else {
397            DeleteMode::Soft
398        };
399
400        // Route index cleanup through the RECORD's namespace, not the caller's.
401        let record_tok = token.with_namespace(
402            khive_types::Namespace::parse(&note.namespace)
403                .map_err(|e| RuntimeError::Internal(format!("note namespace invalid: {e}")))?,
404        );
405        let record_ns = note.namespace.clone();
406        let actor = format!("{}:{}", token.actor().kind, token.actor().id);
407
408        // On hard delete, the row delete and the incident-edge cascade (including
409        // already-soft-deleted edges) run as ONE write transaction: see
410        // `atomic_hard_delete_with_edge_purge`. Index cleanup follows the
411        // commit; it is best-effort and idempotent, unlike the row/edge pair.
412        let deleted = if hard {
413            self.atomic_hard_delete_with_edge_purge(
414                note_hard_delete_statement(id),
415                id,
416                &record_ns,
417                &actor,
418                SubstrateKind::Note,
419            )
420            .await?
421        } else {
422            note_store.delete_note(id, mode).await?
423        };
424        let mut degradations = Vec::new();
425        if deleted {
426            let fts_result = match self.text_for_notes(&record_tok) {
427                Ok(store) => store
428                    .delete_document(&record_ns, id)
429                    .await
430                    .map_err(RuntimeError::from),
431                Err(error) => Err(error),
432            };
433            if let Err(error) = fts_result {
434                record_post_commit_degradation(
435                    &mut degradations,
436                    "delete_note",
437                    id,
438                    "fts_cleanup",
439                    error,
440                );
441            }
442            // Try every model even when FTS or another model failed.
443            for model_name in self.registered_embedding_model_names() {
444                let vector_result = match self.vectors_for_model(&record_tok, &model_name) {
445                    Ok(store) => store.delete(id).await.map_err(RuntimeError::from),
446                    Err(error) => Err(error),
447                };
448                if let Err(error) = vector_result {
449                    record_post_commit_degradation(
450                        &mut degradations,
451                        "delete_note",
452                        id,
453                        "vector_cleanup",
454                        format!("{model_name}: {error}"),
455                    );
456                }
457            }
458            let event = khive_storage::event::Event::new(
459                record_ns.clone(),
460                "delete",
461                EventKind::NoteDeleted,
462                SubstrateKind::Note,
463                "",
464            )
465            .with_target(id)
466            .with_payload(serde_json::json!({"id": id, "namespace": record_ns, "hard": hard}));
467            let event_result = match self.events(&record_tok) {
468                Ok(store) => store.append_event(event).await.map_err(RuntimeError::from),
469                Err(error) => Err(error),
470            };
471            if let Err(error) = event_result {
472                record_post_commit_degradation(
473                    &mut degradations,
474                    "delete_note",
475                    id,
476                    "event_append",
477                    error,
478                );
479            }
480            // A soft OR hard delete removes the note's vectors/FTS document
481            // above: any pack-owned vector-derived cache (e.g.
482            // khive-pack-memory's warm ANN index) needs to know the corpus
483            // changed, reached via this generic hook so khive-runtime never
484            // takes a dependency on khive-pack-memory. No-op when no pack has
485            // installed a hook.
486            self.fire_note_mutation_hook(&note.kind, id).await;
487        }
488        Ok((deleted, degradations))
489    }
490
491    /// Row-first compensating delete for rolling back a partially-written note
492    /// (e.g. `dual_write_message` rollback after a later delivery step fails).
493    /// Unlike [`KhiveRuntime::delete_note`], which cleans up graph/FTS/
494    /// vector indexes *before* removing the row, this removes the row first so
495    /// that a cleanup failure afterward cannot leave the compensated note live.
496    ///
497    /// Returns `Ok(())` once the row is gone (whether or not cleanup fully
498    /// succeeded). Returns `Err(RuntimeError::Internal)` naming the failed
499    /// cleanup legs when row removal succeeded but cleanup did not — the
500    /// message is gone, but stale index entries may remain and should be
501    /// surfaced to the caller rather than silently discarded.
502    ///
503    /// Returns `Ok(())` immediately, with no cleanup attempted, if the note
504    /// does not exist (nothing to compensate).
505    ///
506    /// Not a general-purpose replacement for `delete_note(..., hard=true)`:
507    /// normal hard delete still needs cleanup-first semantics (no dangling
508    /// references) since a caller-visible error there should not remove the row.
509    pub async fn delete_note_row_first_for_compensation(
510        &self,
511        token: &NamespaceToken,
512        id: Uuid,
513    ) -> RuntimeResult<()> {
514        let note_store = self.notes(token)?;
515        let Some(note) = note_store.get_note_including_deleted(id).await? else {
516            return Ok(());
517        };
518        let record_tok = NamespaceToken::for_namespace(
519            khive_types::Namespace::parse(&note.namespace)
520                .map_err(|e| RuntimeError::Internal(format!("note namespace invalid: {e}")))?,
521        );
522        let record_ns = note.namespace.clone();
523
524        // Critical ordering: remove the row before any cleanup that can fail.
525        note_store.delete_note(id, DeleteMode::Hard).await?;
526
527        #[cfg(any(test, feature = "fault-injection"))]
528        {
529            let armed = ROLLBACK_CLEANUP_FAIL_NS.lock().unwrap().take();
530            if armed.as_deref() == Some(record_ns.as_str()) {
531                return Err(RuntimeError::Internal(
532                    "row removed but compensation cleanup failed: injected=true".to_string(),
533                ));
534            }
535        }
536
537        let mut cleanup_errors = Vec::new();
538        if let Err(e) = self.graph(&record_tok)?.purge_incident_edges(id).await {
539            cleanup_errors.push(format!("graph={e}"));
540        }
541        if let Err(e) = self
542            .text_for_notes(&record_tok)?
543            .delete_document(&record_ns, id)
544            .await
545        {
546            cleanup_errors.push(format!("fts={e}"));
547        }
548        for model_name in self.registered_embedding_model_names() {
549            if let Err(e) = self
550                .vectors_for_model(&record_tok, &model_name)?
551                .delete(id)
552                .await
553            {
554                cleanup_errors.push(format!("vector[{model_name}]={e}"));
555            }
556        }
557        if cleanup_errors.is_empty() {
558            Ok(())
559        } else {
560            Err(RuntimeError::Internal(format!(
561                "row removed but compensation cleanup failed: {}",
562                cleanup_errors.join("; ")
563            )))
564        }
565    }
566
567    /// Soft-delete or hard-delete an entity by ID (soft delete by default).
568    ///
569    /// On hard delete, cascades to remove all incident edges (both inbound and
570    /// outbound) to prevent dangling references. Soft delete also cleans FTS
571    /// and vector indexes; edges are left in place.
572    /// Routed attachment cleanup is performed by the registry after ownership resolution.
573    ///
574    /// UUID v4 is globally unique: no namespace filter on by-ID ops.
575    pub async fn delete_entity(
576        &self,
577        token: &NamespaceToken,
578        id: Uuid,
579        hard: bool,
580    ) -> RuntimeResult<bool> {
581        let (deleted, degradations) = self
582            .delete_entity_with_post_commit_report(token, id, hard)
583            .await?;
584        legacy_post_commit_result("delete_entity", id, deleted, degradations)
585    }
586
587    /// Retire one captured Tool registry entity and its prepared authorization rows.
588    ///
589    /// This trusted Rust entry accepts exactly the pack's policy and grant statements;
590    /// their scopes and bindings must be prepared before admission. A stale entity or
591    /// any companion failure rolls back the entire unit. Cleanup errors are reported
592    /// only after confirmed commit and never turn the retirement into a rollback.
593    pub async fn retire_tool_registry_entity_with_post_commit_report(
594        &self,
595        token: &NamespaceToken,
596        expected: &Entity,
597        retired_at: i64,
598        companions: [SqlStatement; 2],
599    ) -> RuntimeResult<([u64; 2], Vec<PostCommitDegradation>)> {
600        if expected.namespace != token.namespace().as_str()
601            || expected.kind != "project"
602            || expected.deleted_at.is_some()
603            || !expected
604                .tags
605                .iter()
606                .any(|tag| khive_types::pack::pack_registry_tag(tag) == Some("tool-registry"))
607        {
608            return Err(RuntimeError::InvalidInput(
609                "retirement requires a live Tool registry row in the write namespace".into(),
610            ));
611        }
612        if retired_at <= expected.updated_at {
613            return Err(RuntimeError::InvalidInput(
614                "registry retirement timestamp must advance the captured row".into(),
615            ));
616        }
617        expected.version.checked_add(1).ok_or_else(|| {
618            RuntimeError::InvalidInput("registry entity version is exhausted".into())
619        })?;
620        let mut retired = expected.clone();
621        retired.deleted_at = Some(retired_at);
622        retired.updated_at = retired_at;
623        crate::secret_gate::reject_reserved_secret_gate_property(retired.properties.as_ref())?;
624        let statement = khive_db::stores::entity::entity_replace_if_unchanged_statement(
625            &retired,
626            expected.updated_at,
627            expected.deleted_at,
628        );
629        let id = expected.id;
630        let result = self
631            .sql()
632            .atomic_unit(Box::new(move |writer| {
633                Box::pin(async move {
634                    if writer.execute(statement).await? != 1 {
635                        return Err(khive_storage::StorageError::Conflict {
636                            capability: khive_storage::StorageCapability::Sql,
637                            operation: "retire_tool_registry_entity".into(),
638                            message: format!(
639                                "entity {id} changed concurrently; retry with fresh state"
640                            ),
641                        });
642                    }
643                    let [policy, grants] = companions;
644                    let counts = [writer.execute(policy).await?, writer.execute(grants).await?];
645                    Ok(Box::new(counts) as Box<dyn std::any::Any + Send>)
646                })
647            }))
648            .await?;
649        let counts = *result
650            .downcast::<[u64; 2]>()
651            .map_err(|_| RuntimeError::Internal("invalid registry retirement result".into()))?;
652        let degradations = self
653            .cleanup_deleted_entity_indexes_and_event(token, &retired, false)
654            .await;
655        Ok((counts, degradations))
656    }
657
658    /// The committed delete together with non-retryable index/telemetry errors.
659    pub async fn delete_entity_with_post_commit_report(
660        &self,
661        token: &NamespaceToken,
662        id: Uuid,
663        hard: bool,
664    ) -> RuntimeResult<(bool, Vec<PostCommitDegradation>)> {
665        let entity = if hard {
666            match self
667                .entities(token)?
668                .get_entity_including_deleted(id)
669                .await?
670            {
671                Some(e) => e,
672                None => return Ok((false, Vec::new())),
673            }
674        } else {
675            match self.entities(token)?.get_entity(id).await? {
676                Some(e) => e,
677                None => return Ok((false, Vec::new())),
678            }
679        };
680        let mode = if hard {
681            DeleteMode::Hard
682        } else {
683            DeleteMode::Soft
684        };
685
686        // Route cascade and index cleanup through the RECORD's namespace, not the caller's.
687        let record_tok = token.with_namespace(
688            khive_types::Namespace::parse(&entity.namespace)
689                .map_err(|e| RuntimeError::Internal(format!("entity namespace invalid: {e}")))?,
690        );
691        let actor = format!("{}:{}", token.actor().kind, token.actor().id);
692
693        // On hard delete, the row delete and the incident-edge cascade (including
694        // already-soft-deleted edges) run as ONE write transaction: see
695        // `atomic_hard_delete_with_edge_purge`. Index cleanup follows the
696        // commit; it is best-effort and idempotent, unlike the row/edge pair.
697        let deleted = if hard {
698            // Cross-backend attachment cleanup requires the registry's ownership check.
699            self.atomic_hard_delete_with_edge_purge(
700                entity_hard_delete_statement(id),
701                id,
702                &entity.namespace,
703                &actor,
704                SubstrateKind::Entity,
705            )
706            .await?
707        } else {
708            self.entities(token)?.delete_entity(id, mode).await?
709        };
710        let degradations = if deleted {
711            self.cleanup_deleted_entity_indexes_and_event(&record_tok, &entity, hard)
712                .await
713        } else {
714            Vec::new()
715        };
716        Ok((deleted, degradations))
717    }
718
719    async fn cleanup_deleted_entity_indexes_and_event(
720        &self,
721        record_tok: &NamespaceToken,
722        entity: &Entity,
723        hard: bool,
724    ) -> Vec<PostCommitDegradation> {
725        let id = entity.id;
726        let mut degradations = Vec::new();
727        let ns = entity.namespace.clone();
728        let fts_result = match self.text(record_tok) {
729            Ok(store) => store
730                .delete_document(&ns, id)
731                .await
732                .map_err(RuntimeError::from),
733            Err(error) => Err(error),
734        };
735        if let Err(error) = fts_result {
736            record_post_commit_degradation(
737                &mut degradations,
738                "delete_entity",
739                id,
740                "fts_cleanup",
741                error,
742            );
743        }
744        for model_name in self.registered_embedding_model_names() {
745            let vector_result = match self.vectors_for_model(record_tok, &model_name) {
746                Ok(store) => store.delete(id).await.map_err(RuntimeError::from),
747                Err(error) => Err(error),
748            };
749            if let Err(error) = vector_result {
750                record_post_commit_degradation(
751                    &mut degradations,
752                    "delete_entity",
753                    id,
754                    "vector_cleanup",
755                    format!("{model_name}: {error}"),
756                );
757            }
758        }
759        let event = khive_storage::event::Event::new(
760            ns.clone(),
761            "delete",
762            EventKind::EntityDeleted,
763            SubstrateKind::Entity,
764            "",
765        )
766        .with_target(id)
767        .with_payload(serde_json::json!({"id": id, "namespace": ns, "hard": hard}));
768        let event_result = match self.events(record_tok) {
769            Ok(store) => store.append_event(event).await.map_err(RuntimeError::from),
770            Err(error) => Err(error),
771        };
772        if let Err(error) = event_result {
773            record_post_commit_degradation(
774                &mut degradations,
775                "delete_entity",
776                id,
777                "event_append",
778                error,
779            );
780        }
781        degradations
782    }
783
784    pub(crate) async fn delete_entity_attachments_on_core(&self, id: Uuid) -> RuntimeResult<bool> {
785        let core = self.core();
786        drop(core.attachments()?);
787        let statement = khive_db::stores::attachment::delete_record_attachments_statement(
788            id,
789            AttachmentSubstrate::Entity,
790        );
791        Ok(core.sql().writer().await?.execute(statement).await? > 0)
792    }
793}
794#[cfg(test)]
795pub(super) mod memory_edge_namespace_restore_tests {
796    use crate::{KhiveRuntime, Resolved, RuntimeError};
797    use khive_storage::{Note, SqlStatement, SqlValue};
798    use khive_types::{EdgeRelation, Namespace};
799    use std::sync::Mutex;
800    use std::time::Duration;
801    use tokio::sync::oneshot;
802    use uuid::Uuid;
803
804    struct BeforeWriterHook {
805        edge_id: Uuid,
806        entered: oneshot::Sender<()>,
807        release: Option<oneshot::Receiver<()>>,
808    }
809
810    static BEFORE_WRITER_HOOKS: Mutex<Vec<BeforeWriterHook>> = Mutex::new(Vec::new());
811
812    fn install_before_writer_hook(
813        edge_id: Uuid,
814        entered: oneshot::Sender<()>,
815        release: Option<oneshot::Receiver<()>>,
816    ) {
817        let mut hooks = BEFORE_WRITER_HOOKS.lock().unwrap();
818        assert!(!hooks.iter().any(|hook| hook.edge_id == edge_id));
819        hooks.push(BeforeWriterHook {
820            edge_id,
821            entered,
822            release,
823        });
824    }
825
826    fn remove_before_writer_hook(edge_id: Uuid) {
827        BEFORE_WRITER_HOOKS
828            .lock()
829            .unwrap()
830            .retain(|hook| hook.edge_id != edge_id);
831    }
832
833    pub(super) async fn before_writer(edge_id: Uuid) {
834        let hook = {
835            let mut hooks = BEFORE_WRITER_HOOKS.lock().unwrap();
836            hooks
837                .iter()
838                .position(|hook| hook.edge_id == edge_id)
839                .map(|index| hooks.remove(index))
840        };
841        if let Some(hook) = hook {
842            let _ = hook.entered.send(());
843            if let Some(release) = hook.release {
844                let _ = release.await;
845            }
846        }
847    }
848
849    fn fixture() -> (KhiveRuntime, tempfile::TempDir) {
850        let dir = tempfile::tempdir().unwrap();
851        let runtime = KhiveRuntime::new_for_test(crate::RuntimeConfig {
852            db_path: Some(dir.path().join("restore-edge-memory-namespace.db")),
853            embedding_model: None,
854            additional_embedding_models: Vec::new(),
855            ..Default::default()
856        })
857        .unwrap();
858        (runtime, dir)
859    }
860
861    async fn execute_private_write(runtime: &KhiveRuntime, statement: SqlStatement) -> u64 {
862        let result = runtime
863            .sql()
864            .atomic_unit(Box::new(move |writer| {
865                Box::pin(async move {
866                    let affected = writer.execute(statement).await?;
867                    Ok(Box::new(affected) as Box<dyn std::any::Any + Send>)
868                })
869            }))
870            .await
871            .unwrap();
872        *result.downcast::<u64>().unwrap()
873    }
874
875    async fn prepare_tombstone(
876        runtime: &KhiveRuntime,
877        edge_id: Uuid,
878        from_namespace: &str,
879        stored_namespace: &str,
880    ) -> u64 {
881        let statement = SqlStatement {
882            sql: "UPDATE graph_edges SET namespace=?1, deleted_at=?2 \
883                  WHERE id=?3 AND namespace=?4 AND deleted_at IS NULL"
884                .into(),
885            params: vec![
886                SqlValue::Text(stored_namespace.into()),
887                SqlValue::Integer(chrono::Utc::now().timestamp_micros()),
888                SqlValue::Text(edge_id.to_string()),
889                SqlValue::Text(from_namespace.into()),
890            ],
891            label: Some("test-prepare-restore-tombstone".into()),
892        };
893        execute_private_write(runtime, statement).await
894    }
895
896    async fn move_memory_source_namespace(
897        runtime: &KhiveRuntime,
898        source_id: Uuid,
899        from_namespace: &str,
900        to_namespace: &str,
901    ) -> u64 {
902        let statement = SqlStatement {
903            sql: "UPDATE notes SET namespace=?1 \
904                  WHERE id=?2 AND namespace=?3 AND kind='memory' AND deleted_at IS NULL"
905                .into(),
906            params: vec![
907                SqlValue::Text(to_namespace.into()),
908                SqlValue::Text(source_id.to_string()),
909                SqlValue::Text(from_namespace.into()),
910            ],
911            label: Some("test-move-memory-source-between-check-and-restore".into()),
912        };
913        execute_private_write(runtime, statement).await
914    }
915
916    async fn soft_delete_memory_source(runtime: &KhiveRuntime, source_id: Uuid) -> u64 {
917        let statement = SqlStatement {
918            sql: "UPDATE notes SET deleted_at=?1 \
919                  WHERE id=?2 AND namespace=?3 AND kind='memory' AND deleted_at IS NULL"
920                .into(),
921            params: vec![
922                SqlValue::Integer(chrono::Utc::now().timestamp_micros()),
923                SqlValue::Text(source_id.to_string()),
924                SqlValue::Text("local".into()),
925            ],
926            label: Some("test-soft-delete-memory-source-between-check-and-restore".into()),
927        };
928        execute_private_write(runtime, statement).await
929    }
930
931    async fn change_note_kind(
932        runtime: &KhiveRuntime,
933        note_id: Uuid,
934        from_kind: &str,
935        to_kind: &str,
936    ) -> u64 {
937        let statement = SqlStatement {
938            sql: "UPDATE notes SET kind=?1 WHERE id=?2 AND kind=?3 AND deleted_at IS NULL".into(),
939            params: vec![
940                SqlValue::Text(to_kind.into()),
941                SqlValue::Text(note_id.to_string()),
942                SqlValue::Text(from_kind.into()),
943            ],
944            label: Some("test-promote-target-to-memory-between-check-and-restore".into()),
945        };
946        execute_private_write(runtime, statement).await
947    }
948
949    async fn assert_memory_note_in_namespace(
950        runtime: &KhiveRuntime,
951        token: &crate::NamespaceToken,
952        id: Uuid,
953        expected_namespace: &str,
954    ) {
955        match runtime
956            .resolve_by_id_including_deleted(token, id)
957            .await
958            .unwrap()
959        {
960            Some(Resolved::Note(note)) => {
961                assert_eq!(note.namespace, expected_namespace);
962                assert_eq!(note.kind, "memory");
963                assert!(note.deleted_at.is_none());
964            }
965            _ => panic!("fixture endpoint was not a live memory note"),
966        }
967    }
968
969    fn assert_helper_namespace_error(message: &str) {
970        assert_eq!(
971            message,
972            "restore: memory edge source namespace other does not match stored edge namespace local; leave this tombstone deleted and create a replacement link in namespace=other",
973            "restore_refusal_names_operation_and_supported_remedy"
974        );
975    }
976
977    #[tokio::test]
978    async fn restore_edge_rejects_legacy_tombstone_with_mismatched_memory_source_namespace() {
979        let (runtime, _dir) = fixture();
980        // Keep the source namespace readable for this diagnostic assertion.
981        let local_token = runtime
982            .authorize_with_visibility(Namespace::local(), vec![Namespace::parse("other").unwrap()])
983            .unwrap();
984        let other_token = runtime
985            .authorize(Namespace::parse("other").unwrap())
986            .unwrap();
987
988        let source = Note::new("other", "memory", "restore source");
989        let source_id = source.id;
990        let target = Note::new("other", "memory", "restore target");
991        let target_id = target.id;
992        runtime
993            .notes(&other_token)
994            .unwrap()
995            .upsert_note(source)
996            .await
997            .unwrap();
998        runtime
999            .notes(&other_token)
1000            .unwrap()
1001            .upsert_note(target)
1002            .await
1003            .unwrap();
1004
1005        let edge = runtime
1006            .link(
1007                &other_token,
1008                source_id,
1009                target_id,
1010                EdgeRelation::Annotates,
1011                1.0,
1012                None,
1013            )
1014            .await
1015            .unwrap();
1016        let edge_id = Uuid::from(edge.id);
1017        assert_eq!(edge.namespace, "other");
1018        assert_eq!(edge.source_id, source_id);
1019        assert_eq!(edge.target_id, target_id);
1020        assert_eq!(edge.relation, EdgeRelation::Annotates);
1021
1022        assert_eq!(
1023            prepare_tombstone(&runtime, edge_id, "other", "local").await,
1024            1
1025        );
1026        let stored_edge = runtime
1027            .get_edge_including_deleted(&local_token, edge_id)
1028            .await
1029            .unwrap()
1030            .expect("synthetic legacy edge row should remain addressable");
1031        assert_eq!(stored_edge.namespace, "local");
1032        assert_eq!(stored_edge.source_id, source_id);
1033        assert_eq!(stored_edge.target_id, target_id);
1034        assert_eq!(stored_edge.relation, EdgeRelation::Annotates);
1035        assert!(stored_edge.deleted_at.is_some());
1036        assert_memory_note_in_namespace(&runtime, &local_token, source_id, "other").await;
1037        assert_memory_note_in_namespace(&runtime, &local_token, target_id, "other").await;
1038
1039        let (entered_tx, mut entered_rx) = oneshot::channel();
1040        install_before_writer_hook(edge_id, entered_tx, None);
1041        let outcome = runtime.restore_edge(&local_token, edge_id).await;
1042        remove_before_writer_hook(edge_id);
1043        let writer_boundary_reached = matches!(entered_rx.try_recv(), Ok(()));
1044        match outcome {
1045            Err(RuntimeError::InvalidInput(message)) => {
1046                if writer_boundary_reached {
1047                    panic!("restore_memory_namespace_guard_bypassed_writer_reached");
1048                }
1049                assert_helper_namespace_error(&message);
1050            }
1051            Ok(Some((_, true))) => panic!("restore_memory_namespace_mismatch_admitted"),
1052            _ => panic!("fixture did not reach the targeted restore refusal"),
1053        }
1054
1055        let after_refusal = runtime
1056            .get_edge_including_deleted(&local_token, edge_id)
1057            .await
1058            .unwrap()
1059            .expect("refused restore must preserve the tombstone row");
1060        assert_eq!(after_refusal.namespace, "local");
1061        assert!(after_refusal.deleted_at.is_some());
1062    }
1063
1064    #[tokio::test]
1065    async fn restore_edge_rechecks_memory_source_namespace_at_writer_after_precheck() {
1066        let (runtime, _dir) = fixture();
1067        let local_token = runtime.authorize(Namespace::local()).unwrap();
1068        let other_token = runtime
1069            .authorize(Namespace::parse("other").unwrap())
1070            .unwrap();
1071
1072        let source = Note::new("local", "memory", "moving restore source");
1073        let source_id = source.id;
1074        let target = Note::new("local", "memory", "stable restore target");
1075        let target_id = target.id;
1076        runtime
1077            .notes(&local_token)
1078            .unwrap()
1079            .upsert_note(source)
1080            .await
1081            .unwrap();
1082        runtime
1083            .notes(&local_token)
1084            .unwrap()
1085            .upsert_note(target)
1086            .await
1087            .unwrap();
1088
1089        let edge = runtime
1090            .link(
1091                &local_token,
1092                source_id,
1093                target_id,
1094                EdgeRelation::Annotates,
1095                1.0,
1096                None,
1097            )
1098            .await
1099            .unwrap();
1100        let edge_id = Uuid::from(edge.id);
1101        assert_eq!(edge.namespace, "local");
1102        assert_eq!(edge.source_id, source_id);
1103        assert_eq!(edge.target_id, target_id);
1104        assert_eq!(edge.relation, EdgeRelation::Annotates);
1105        assert_eq!(
1106            prepare_tombstone(&runtime, edge_id, "local", "local").await,
1107            1
1108        );
1109        let edge_before_restore = runtime
1110            .get_edge_including_deleted(&local_token, edge_id)
1111            .await
1112            .unwrap()
1113            .expect("tombstone should be present before restore");
1114        assert_eq!(edge_before_restore.namespace, "local");
1115        assert!(edge_before_restore.deleted_at.is_some());
1116        assert_memory_note_in_namespace(&runtime, &local_token, source_id, "local").await;
1117        assert_memory_note_in_namespace(&runtime, &local_token, target_id, "local").await;
1118
1119        let (entered_tx, mut entered_rx) = oneshot::channel();
1120        let (release_tx, release_rx) = oneshot::channel();
1121        install_before_writer_hook(edge_id, entered_tx, Some(release_rx));
1122        let mut restore = Box::pin(runtime.restore_edge(&local_token, edge_id));
1123        let crossed_precheck = tokio::time::timeout(Duration::from_secs(10), async {
1124            tokio::select! {
1125                _ = &mut restore => false,
1126                reached = &mut entered_rx => reached.is_ok(),
1127            }
1128        })
1129        .await
1130        .unwrap_or(false);
1131        if !crossed_precheck {
1132            remove_before_writer_hook(edge_id);
1133            panic!("fixture did not positively cross the outside guard before writer");
1134        }
1135
1136        assert_eq!(
1137            move_memory_source_namespace(&runtime, source_id, "local", "other").await,
1138            1
1139        );
1140        assert_memory_note_in_namespace(&runtime, &other_token, source_id, "other").await;
1141        assert_memory_note_in_namespace(&runtime, &local_token, target_id, "local").await;
1142        let edge_while_paused = runtime
1143            .get_edge_including_deleted(&local_token, edge_id)
1144            .await
1145            .unwrap()
1146            .expect("edge must still exist while restore is held before writer");
1147        assert_eq!(edge_while_paused.namespace, "local");
1148        assert!(edge_while_paused.deleted_at.is_some());
1149
1150        assert!(release_tx.send(()).is_ok());
1151        let outcome = tokio::time::timeout(Duration::from_secs(10), &mut restore)
1152            .await
1153            .expect("restore did not complete after release");
1154        match outcome {
1155            Err(RuntimeError::InvalidInput(_)) => {}
1156            Ok(Some((_, true))) => {
1157                panic!("restore_memory_namespace_drift_admitted_after_precheck")
1158            }
1159            _ => panic!("writer-fence fixture did not return the expected refusal class"),
1160        }
1161
1162        let after_refusal = runtime
1163            .get_edge_including_deleted(&local_token, edge_id)
1164            .await
1165            .unwrap()
1166            .expect("writer-fence refusal must preserve the tombstone");
1167        assert_eq!(after_refusal.namespace, "local");
1168        assert!(after_refusal.deleted_at.is_some());
1169    }
1170
1171    #[tokio::test]
1172    async fn restore_edge_rechecks_memory_source_liveness_at_writer_after_precheck() {
1173        let (runtime, _dir) = fixture();
1174        let local_token = runtime.authorize(Namespace::local()).unwrap();
1175
1176        let source = Note::new("local", "memory", "soft-deleted restore source");
1177        let source_id = source.id;
1178        let target = Note::new("local", "memory", "live restore target");
1179        let target_id = target.id;
1180        runtime
1181            .notes(&local_token)
1182            .unwrap()
1183            .upsert_note(source)
1184            .await
1185            .unwrap();
1186        runtime
1187            .notes(&local_token)
1188            .unwrap()
1189            .upsert_note(target)
1190            .await
1191            .unwrap();
1192
1193        let edge = runtime
1194            .link(
1195                &local_token,
1196                source_id,
1197                target_id,
1198                EdgeRelation::Annotates,
1199                1.0,
1200                None,
1201            )
1202            .await
1203            .unwrap();
1204        let edge_id = Uuid::from(edge.id);
1205        assert_eq!(edge.namespace, "local");
1206        assert_eq!(
1207            prepare_tombstone(&runtime, edge_id, "local", "local").await,
1208            1
1209        );
1210        assert_memory_note_in_namespace(&runtime, &local_token, source_id, "local").await;
1211        assert_memory_note_in_namespace(&runtime, &local_token, target_id, "local").await;
1212
1213        let (entered_tx, mut entered_rx) = oneshot::channel();
1214        let (release_tx, release_rx) = oneshot::channel();
1215        install_before_writer_hook(edge_id, entered_tx, Some(release_rx));
1216        let mut restore = Box::pin(runtime.restore_edge(&local_token, edge_id));
1217        let crossed_precheck = tokio::time::timeout(Duration::from_secs(10), async {
1218            tokio::select! {
1219                _ = &mut restore => false,
1220                reached = &mut entered_rx => reached.is_ok(),
1221            }
1222        })
1223        .await
1224        .unwrap_or(false);
1225        if !crossed_precheck {
1226            remove_before_writer_hook(edge_id);
1227            panic!("fixture did not positively cross the outside guard before writer");
1228        }
1229
1230        assert_eq!(soft_delete_memory_source(&runtime, source_id).await, 1);
1231        match runtime
1232            .resolve_by_id_including_deleted(&local_token, source_id)
1233            .await
1234            .unwrap()
1235        {
1236            Some(Resolved::Note(note)) => {
1237                assert_eq!(note.namespace, "local");
1238                assert_eq!(note.kind, "memory");
1239                assert!(note.deleted_at.is_some());
1240            }
1241            _ => panic!("fixture source did not remain as an included-deleted memory note"),
1242        }
1243        assert_memory_note_in_namespace(&runtime, &local_token, target_id, "local").await;
1244
1245        assert!(release_tx.send(()).is_ok());
1246        let outcome = tokio::time::timeout(Duration::from_secs(10), &mut restore)
1247            .await
1248            .expect("restore did not complete after release");
1249        match outcome {
1250            Err(RuntimeError::InvalidInput(_)) => {}
1251            Ok(Some((_, true))) => {
1252                panic!("restore_memory_source_soft_delete_admitted_after_precheck")
1253            }
1254            _ => panic!("source-liveness fence fixture did not return the expected refusal class"),
1255        }
1256        let after_refusal = runtime
1257            .get_edge_including_deleted(&local_token, edge_id)
1258            .await
1259            .unwrap()
1260            .expect("source-liveness refusal must preserve the tombstone");
1261        assert_eq!(after_refusal.namespace, "local");
1262        assert!(after_refusal.deleted_at.is_some());
1263    }
1264
1265    #[tokio::test]
1266    async fn restore_edge_rejects_preexisting_soft_deleted_memory_source() {
1267        let (runtime, _dir) = fixture();
1268        let local_token = runtime.authorize(Namespace::local()).unwrap();
1269        let source = Note::new("local", "memory", "already deleted restore source");
1270        let source_id = source.id;
1271        let target = Note::new("local", "memory", "live restore target");
1272        let target_id = target.id;
1273        runtime
1274            .notes(&local_token)
1275            .unwrap()
1276            .upsert_note(source)
1277            .await
1278            .unwrap();
1279        runtime
1280            .notes(&local_token)
1281            .unwrap()
1282            .upsert_note(target)
1283            .await
1284            .unwrap();
1285
1286        let edge = runtime
1287            .link(
1288                &local_token,
1289                source_id,
1290                target_id,
1291                EdgeRelation::Annotates,
1292                1.0,
1293                None,
1294            )
1295            .await
1296            .unwrap();
1297        let edge_id = Uuid::from(edge.id);
1298        assert_eq!(
1299            prepare_tombstone(&runtime, edge_id, "local", "local").await,
1300            1
1301        );
1302        let stored_edge = runtime
1303            .get_edge_including_deleted(&local_token, edge_id)
1304            .await
1305            .unwrap()
1306            .expect("fixture tombstone should be addressable before source deletion");
1307        assert_eq!(stored_edge.namespace, "local");
1308        assert!(stored_edge.deleted_at.is_some());
1309        assert_eq!(soft_delete_memory_source(&runtime, source_id).await, 1);
1310        match runtime
1311            .resolve_by_id_including_deleted(&local_token, source_id)
1312            .await
1313            .unwrap()
1314        {
1315            Some(Resolved::Note(note)) => {
1316                assert_eq!(note.namespace, "local");
1317                assert_eq!(note.kind, "memory");
1318                assert!(note.deleted_at.is_some());
1319            }
1320            _ => panic!("fixture source did not remain as an included-deleted memory note"),
1321        }
1322        assert_memory_note_in_namespace(&runtime, &local_token, target_id, "local").await;
1323
1324        let (entered_tx, mut entered_rx) = oneshot::channel();
1325        install_before_writer_hook(edge_id, entered_tx, None);
1326        let outcome = runtime.restore_edge(&local_token, edge_id).await;
1327        remove_before_writer_hook(edge_id);
1328        assert!(
1329            matches!(entered_rx.try_recv(), Ok(())),
1330            "fixture did not reach writer fence"
1331        );
1332        match outcome {
1333            Err(RuntimeError::InvalidInput(_)) => {}
1334            Ok(Some((_, true))) => panic!("restore_memory_predeleted_source_admitted"),
1335            _ => panic!("predeleted-source fixture did not return the expected refusal class"),
1336        }
1337        let after_refusal = runtime
1338            .get_edge_including_deleted(&local_token, edge_id)
1339            .await
1340            .unwrap()
1341            .expect("predeleted-source refusal must preserve the tombstone");
1342        assert!(after_refusal.deleted_at.is_some());
1343    }
1344
1345    #[tokio::test]
1346    async fn restore_edge_rechecks_current_memory_pair_after_nonmemory_precheck() {
1347        let (runtime, _dir) = fixture();
1348        let local_token = runtime.authorize(Namespace::local()).unwrap();
1349        let other_token = runtime
1350            .authorize(Namespace::parse("other").unwrap())
1351            .unwrap();
1352        let source = Note::new(
1353            "other",
1354            "memory",
1355            "memory source before current-pair change",
1356        );
1357        let source_id = source.id;
1358        let target = Note::new(
1359            "other",
1360            "observation",
1361            "nonmemory target before current-pair change",
1362        );
1363        let target_id = target.id;
1364        runtime
1365            .notes(&other_token)
1366            .unwrap()
1367            .upsert_note(source)
1368            .await
1369            .unwrap();
1370        runtime
1371            .notes(&other_token)
1372            .unwrap()
1373            .upsert_note(target)
1374            .await
1375            .unwrap();
1376
1377        let edge = runtime
1378            .link(
1379                &other_token,
1380                source_id,
1381                target_id,
1382                EdgeRelation::Annotates,
1383                1.0,
1384                None,
1385            )
1386            .await
1387            .unwrap();
1388        let edge_id = Uuid::from(edge.id);
1389        assert_eq!(
1390            prepare_tombstone(&runtime, edge_id, "other", "local").await,
1391            1
1392        );
1393        let edge_before_restore = runtime
1394            .get_edge_including_deleted(&local_token, edge_id)
1395            .await
1396            .unwrap()
1397            .expect("current-pair fixture tombstone should be addressable");
1398        assert_eq!(edge_before_restore.namespace, "local");
1399        assert_eq!(edge_before_restore.source_id, source_id);
1400        assert_eq!(edge_before_restore.target_id, target_id);
1401        assert!(edge_before_restore.deleted_at.is_some());
1402        assert_memory_note_in_namespace(&runtime, &local_token, source_id, "other").await;
1403        match runtime
1404            .resolve_by_id_including_deleted(&local_token, target_id)
1405            .await
1406            .unwrap()
1407        {
1408            Some(Resolved::Note(note)) => {
1409                assert_eq!(note.namespace, "other");
1410                assert_eq!(note.kind, "observation");
1411                assert!(note.deleted_at.is_none());
1412            }
1413            _ => panic!("fixture target was not the expected nonmemory Note"),
1414        }
1415
1416        match runtime.restore_edge(&local_token, edge_id).await {
1417            Ok(Some((restored, true))) => {
1418                assert_eq!(restored.namespace, "local");
1419                assert!(restored.deleted_at.is_none());
1420            }
1421            _ => panic!("ordinary memory-to-observation restore should preserve existing behavior"),
1422        }
1423        let ordinary_restore = runtime
1424            .get_edge_including_deleted(&local_token, edge_id)
1425            .await
1426            .unwrap()
1427            .expect("ordinary nonmemory-pair restore should remain readable");
1428        assert_eq!(ordinary_restore.namespace, "local");
1429        assert!(ordinary_restore.deleted_at.is_none());
1430        assert_eq!(
1431            prepare_tombstone(&runtime, edge_id, "local", "local").await,
1432            1
1433        );
1434        let edge_before_race = runtime
1435            .get_edge_including_deleted(&local_token, edge_id)
1436            .await
1437            .unwrap()
1438            .expect("retombstoned edge should be addressable before current-pair race");
1439        assert_eq!(edge_before_race.namespace, "local");
1440        assert!(edge_before_race.deleted_at.is_some());
1441
1442        let (entered_tx, mut entered_rx) = oneshot::channel();
1443        let (release_tx, release_rx) = oneshot::channel();
1444        install_before_writer_hook(edge_id, entered_tx, Some(release_rx));
1445        let mut restore = Box::pin(runtime.restore_edge(&local_token, edge_id));
1446        let crossed_precheck = tokio::time::timeout(Duration::from_secs(10), async {
1447            tokio::select! {
1448                _ = &mut restore => false,
1449                reached = &mut entered_rx => reached.is_ok(),
1450            }
1451        })
1452        .await
1453        .unwrap_or(false);
1454        if !crossed_precheck {
1455            remove_before_writer_hook(edge_id);
1456            panic!("fixture did not positively cross the outside check before writer");
1457        }
1458
1459        assert_eq!(
1460            change_note_kind(&runtime, target_id, "observation", "memory").await,
1461            1
1462        );
1463        match runtime
1464            .resolve_by_id_including_deleted(&local_token, target_id)
1465            .await
1466            .unwrap()
1467        {
1468            Some(Resolved::Note(note)) => {
1469                assert_eq!(note.namespace, "other");
1470                assert_eq!(note.kind, "memory");
1471                assert!(note.deleted_at.is_none());
1472            }
1473            _ => panic!("fixture target did not become a live memory Note"),
1474        }
1475        assert!(release_tx.send(()).is_ok());
1476        let outcome = tokio::time::timeout(Duration::from_secs(10), &mut restore)
1477            .await
1478            .expect("restore did not complete after release");
1479        match outcome {
1480            Err(RuntimeError::InvalidInput(_)) => {}
1481            Ok(Some((_, true))) => {
1482                panic!("restore_current_memory_pair_source_namespace_admitted")
1483            }
1484            _ => panic!("current-pair fixture did not return the expected refusal class"),
1485        }
1486        let after_refusal = runtime
1487            .get_edge_including_deleted(&local_token, edge_id)
1488            .await
1489            .unwrap()
1490            .expect("current-pair refusal must preserve the tombstone");
1491        assert_eq!(after_refusal.namespace, "local");
1492        assert!(after_refusal.deleted_at.is_some());
1493    }
1494
1495    #[tokio::test]
1496    async fn restore_edge_allows_memory_target_in_another_namespace() {
1497        let (runtime, _dir) = fixture();
1498        let local_token = runtime.authorize(Namespace::local()).unwrap();
1499        let other_token = runtime
1500            .authorize(Namespace::parse("other").unwrap())
1501            .unwrap();
1502
1503        let source = Note::new("local", "memory", "local restore source");
1504        let source_id = source.id;
1505        let target = Note::new("other", "memory", "foreign restore target");
1506        let target_id = target.id;
1507        runtime
1508            .notes(&local_token)
1509            .unwrap()
1510            .upsert_note(source)
1511            .await
1512            .unwrap();
1513        runtime
1514            .notes(&other_token)
1515            .unwrap()
1516            .upsert_note(target)
1517            .await
1518            .unwrap();
1519
1520        let edge = runtime
1521            .link(
1522                &local_token,
1523                source_id,
1524                target_id,
1525                EdgeRelation::Annotates,
1526                1.0,
1527                None,
1528            )
1529            .await
1530            .unwrap();
1531        let edge_id = Uuid::from(edge.id);
1532        assert_eq!(edge.namespace, "local");
1533        assert_eq!(edge.source_id, source_id);
1534        assert_eq!(edge.target_id, target_id);
1535        assert_eq!(
1536            prepare_tombstone(&runtime, edge_id, "local", "local").await,
1537            1
1538        );
1539        assert_memory_note_in_namespace(&runtime, &local_token, source_id, "local").await;
1540        assert_memory_note_in_namespace(&runtime, &local_token, target_id, "other").await;
1541
1542        match runtime.restore_edge(&local_token, edge_id).await {
1543            Ok(Some((restored, true))) => {
1544                assert_eq!(restored.namespace, "local");
1545                assert!(restored.deleted_at.is_none());
1546            }
1547            _ => panic!("source-local memory edge with a cross-namespace target should restore"),
1548        }
1549        let after_restore = runtime
1550            .get_edge_including_deleted(&local_token, edge_id)
1551            .await
1552            .unwrap()
1553            .expect("restored edge should remain readable");
1554        assert_eq!(after_restore.namespace, "local");
1555        assert!(after_restore.deleted_at.is_none());
1556    }
1557
1558    #[tokio::test]
1559    async fn restore_refusal_hides_unreadable_memory_source_namespace() {
1560        let (runtime, _dir) = fixture();
1561        let source_namespace = "guard-private-source";
1562        let source_token = runtime
1563            .authorize(Namespace::parse(source_namespace).unwrap())
1564            .unwrap();
1565        let readable = runtime
1566            .authorize_with_visibility(
1567                Namespace::local(),
1568                vec![Namespace::parse(source_namespace).unwrap()],
1569            )
1570            .unwrap();
1571        let unreadable = runtime
1572            .authorize_with_visibility(Namespace::local(), Vec::new())
1573            .unwrap();
1574        assert!(readable
1575            .visible_namespaces()
1576            .iter()
1577            .any(|ns| ns.as_str() == source_namespace));
1578        assert!(!unreadable
1579            .visible_namespaces()
1580            .iter()
1581            .any(|ns| ns.as_str() == source_namespace));
1582        let source = Note::new(source_namespace, "memory", "source");
1583        let target = Note::new(source_namespace, "memory", "target");
1584        let (source_id, target_id) = (source.id, target.id);
1585        runtime
1586            .notes(&source_token)
1587            .unwrap()
1588            .upsert_note(source)
1589            .await
1590            .unwrap();
1591        runtime
1592            .notes(&source_token)
1593            .unwrap()
1594            .upsert_note(target)
1595            .await
1596            .unwrap();
1597        assert_memory_note_in_namespace(&runtime, &source_token, source_id, source_namespace).await;
1598        assert_memory_note_in_namespace(&runtime, &source_token, target_id, source_namespace).await;
1599        let edge = runtime
1600            .link(
1601                &source_token,
1602                source_id,
1603                target_id,
1604                EdgeRelation::Annotates,
1605                1.0,
1606                None,
1607            )
1608            .await
1609            .expect("matching source namespace must admit the fixture link");
1610        assert_eq!(edge.namespace.as_str(), source_namespace);
1611        let edge_id = Uuid::from(edge.id);
1612        assert_eq!(
1613            prepare_tombstone(&runtime, edge_id, source_namespace, "local").await,
1614            1
1615        );
1616        let before = runtime
1617            .get_edge_including_deleted(&unreadable, edge_id)
1618            .await
1619            .unwrap()
1620            .unwrap();
1621        assert_eq!(before.namespace.as_str(), "local");
1622        assert!(before.deleted_at.is_some());
1623        let RuntimeError::InvalidInput(message) =
1624            runtime.restore_edge(&readable, edge_id).await.unwrap_err()
1625        else {
1626            panic!("readable mismatch must preserve InvalidInput");
1627        };
1628        assert_eq!(message, format!("restore: memory edge source namespace {source_namespace} does not match stored edge namespace local; leave this tombstone deleted and create a replacement link in namespace={source_namespace}"));
1629        let RuntimeError::InvalidInput(message) = runtime
1630            .restore_edge(&unreadable, edge_id)
1631            .await
1632            .unwrap_err()
1633        else {
1634            panic!("unreadable mismatch must preserve InvalidInput");
1635        };
1636        assert!(
1637            !message.contains(source_namespace),
1638            "restore_unreadable_source_namespace_not_disclosed"
1639        );
1640        assert!(!message.contains(&source_id.to_string()));
1641        assert!(!message.contains(&target_id.to_string()));
1642        assert_eq!(message, "restore: memory edge source row is in a namespace the caller cannot read; the edge cannot be restored under stored edge namespace local");
1643        let after = runtime
1644            .get_edge_including_deleted(&unreadable, edge_id)
1645            .await
1646            .unwrap()
1647            .unwrap();
1648        assert_eq!(after.namespace.as_str(), "local");
1649        assert_eq!(after.source_id, source_id);
1650        assert_eq!(after.target_id, target_id);
1651        assert!(after.deleted_at.is_some());
1652    }
1653}