Skip to main content

rigspark_runtime/
memory.rs

1#[cfg(test)]
2mod recovery_tests {
3    use super::*;
4    #[test]
5    fn interrupted_move_restores_source_and_target_from_either_entry_point() {
6        for open_source in [true, false] {
7            let root = tempfile::tempdir().unwrap();
8            let source = MemoryStore::open(root.path(), "source", "original-source").unwrap();
9            let target = MemoryStore::open(root.path(), "target", "original-target").unwrap();
10            let expected = target.files().unwrap();
11            let mut next = expected.clone();
12            next.insert("system.md".into(), "migrated persona".into());
13            let guard = source.lock().unwrap();
14            target.publish_pending(&guard, &expected, &next).unwrap();
15            target.prepare_move(&source, &guard).unwrap();
16            guard.release().unwrap();
17            if open_source {
18                MemoryStore::existing(root.path(), "source").unwrap();
19            } else {
20                MemoryStore::existing(root.path(), "target").unwrap();
21            }
22            assert_eq!(
23                MemoryStore::existing(root.path(), "source")
24                    .unwrap()
25                    .meta
26                    .created_at,
27                "original-source"
28            );
29            assert_eq!(
30                MemoryStore::existing(root.path(), "target")
31                    .unwrap()
32                    .files()
33                    .unwrap(),
34                expected
35            );
36        }
37    }
38}
39use crate::state::{Config, LockGuard, StateStore};
40use serde::{Deserialize, Serialize};
41use sha2::{Digest, Sha256};
42use std::{
43    fs,
44    io::Write,
45    path::{Path, PathBuf},
46    time::Duration,
47};
48
49#[derive(Debug, thiserror::Error)]
50#[error("memory: {0}")]
51pub struct MemoryError(pub String);
52impl From<std::io::Error> for MemoryError {
53    fn from(error: std::io::Error) -> Self {
54        Self(error.to_string())
55    }
56}
57impl From<serde_json::Error> for MemoryError {
58    fn from(error: serde_json::Error) -> Self {
59        Self(error.to_string())
60    }
61}
62impl From<crate::state::StateError> for MemoryError {
63    fn from(error: crate::state::StateError) -> Self {
64        Self(error.to_string())
65    }
66}
67#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
68#[serde(deny_unknown_fields)]
69pub struct EmbeddingMeta {
70    pub model: String,
71    pub dimension: usize,
72}
73#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
74#[serde(rename_all = "camelCase", deny_unknown_fields)]
75pub struct MemoryMeta {
76    pub schema_version: u8,
77    pub model_id: String,
78    pub created_at: String,
79    #[serde(skip_serializing_if = "Option::is_none")]
80    pub embedding: Option<EmbeddingMeta>,
81    #[serde(skip_serializing_if = "Option::is_none")]
82    pub embedding_unsupported: Option<bool>,
83}
84impl MemoryMeta {
85    pub fn validate(&self) -> Result<(), MemoryError> {
86        if self.schema_version != 1
87            || self.model_id.is_empty()
88            || self.model_id.len() > 8192
89            || self.created_at.is_empty()
90            || self.created_at.len() > 1024
91            || self.embedding_unsupported == Some(false)
92            || self.embedding.as_ref().is_some_and(|embedding| {
93                embedding.model.is_empty()
94                    || embedding.model.len() > 8192
95                    || embedding.dimension == 0
96                    || embedding.dimension > 100000
97            })
98        {
99            return Err(MemoryError("invalid metadata".into()));
100        }
101        Ok(())
102    }
103}
104pub fn memory_slug(id: &str, windows: bool) -> Result<String, MemoryError> {
105    if id.len() > 1024 * 1024 {
106        return Err(MemoryError("model id exceeds limit".into()));
107    }
108    let mut slug = String::new();
109    for character in id.trim().to_lowercase().chars() {
110        let character = if character.is_ascii_alphanumeric() || "._-".contains(character) {
111            character
112        } else {
113            '-'
114        };
115        if character != '-' || !slug.ends_with('-') {
116            slug.push(character);
117        }
118    }
119    slug = slug.trim_matches(['-', '.']).into();
120    let stem = slug.split('.').next().unwrap_or("");
121    if windows
122        && (["con", "prn", "aux", "nul"].contains(&stem)
123            || (stem.starts_with("com") || stem.starts_with("lpt"))
124                && stem.len() == 4
125                && matches!(stem.as_bytes()[3], b'1'..=b'9'))
126    {
127        slug = format!("x-{slug}");
128    }
129    if slug.is_empty() {
130        return Err(MemoryError("model id has no filesystem-safe slug".into()));
131    }
132    if slug.len() > 128 {
133        let digest = format!("{:x}", Sha256::digest(slug.as_bytes()));
134        slug = format!(
135            "{}-{}",
136            slug[..111].trim_end_matches(['-', '.']),
137            &digest[..16]
138        );
139    }
140    Ok(slug)
141}
142pub(crate) fn private_directory(path: &Path) -> Result<(), MemoryError> {
143    match fs::symlink_metadata(path) {
144        Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => (),
145        Ok(_) => {
146            return Err(MemoryError(
147                "directory is symlinked or not a directory".into(),
148            ));
149        }
150        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
151            if let Some(parent) = path.parent() {
152                private_directory(parent)?;
153            }
154            let builder = fs::DirBuilder::new();
155            #[cfg(unix)]
156            let mut builder = builder;
157            #[cfg(unix)]
158            {
159                use std::os::unix::fs::DirBuilderExt;
160                builder.mode(0o700);
161            }
162            match builder.create(path) {
163                Ok(()) => (),
164                Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
165                    return private_directory(path);
166                }
167                Err(error) => return Err(error.into()),
168            }
169        }
170        Err(error) => return Err(error.into()),
171    }
172    Ok(())
173}
174pub(crate) fn owned_directory(path: &Path) -> Result<(), MemoryError> {
175    private_directory(path)?;
176    #[cfg(unix)]
177    {
178        use std::os::unix::fs::PermissionsExt;
179        crate::secure_fs::Directory::open(path)?
180            .dir()?
181            .open(".")?
182            .into_std()
183            .set_permissions(fs::Permissions::from_mode(0o700))?;
184    }
185    Ok(())
186}
187pub(crate) fn atomic_file(path: &Path, bytes: &[u8]) -> Result<(), MemoryError> {
188    let parent = path
189        .parent()
190        .ok_or_else(|| MemoryError("missing file parent".into()))?;
191    private_directory(parent)?;
192    let mut staged = tempfile::NamedTempFile::new_in(parent)?;
193    staged.write_all(bytes)?;
194    staged.as_file().sync_all()?;
195    staged
196        .persist(path)
197        .map_err(|error| MemoryError(error.to_string()))?;
198    #[cfg(unix)]
199    fs::File::open(parent)?.sync_all()?;
200    Ok(())
201}
202#[derive(Debug, Clone)]
203pub struct MemoryStore {
204    pub dir: PathBuf,
205    pub meta: MemoryMeta,
206    home: PathBuf,
207    home_directory: std::sync::Arc<crate::secure_fs::Directory>,
208}
209#[derive(Serialize, Deserialize)]
210#[serde(deny_unknown_fields)]
211struct MoveJournal {
212    source: String,
213    recovery: String,
214}
215impl MemoryStore {
216    pub fn open(home: &Path, id: &str, created_at: &str) -> Result<Self, MemoryError> {
217        let home = std::path::absolute(home)?;
218        let root = home.join("memory");
219        owned_directory(&home)?;
220        let home_directory = std::sync::Arc::new(crate::secure_fs::Directory::open(&home)?);
221        home_directory.ensure_dir(Path::new("memory"))?;
222        let dir = root.join(memory_slug(id, cfg!(windows))?);
223        let store = Self {
224            dir,
225            home,
226            home_directory,
227            meta: MemoryMeta {
228                schema_version: 1,
229                model_id: id.into(),
230                created_at: created_at.into(),
231                embedding: None,
232                embedding_unsupported: None,
233            },
234        };
235        store.initialize()
236    }
237    fn initialize(self) -> Result<Self, MemoryError> {
238        let mut store = self;
239        store.recover_if_needed()?;
240        store.mkdir(&store.dir)?;
241        let path = store.dir.join("meta.json");
242        if !path.try_exists()? {
243            store.meta.validate()?;
244            let mut bytes = serde_json::to_vec_pretty(&store.meta)?;
245            bytes.push(b'\n');
246            match store
247                .home_directory
248                .write(store.relative(&path)?, &bytes, true, false)
249            {
250                Ok(()) => (),
251                Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => (),
252                Err(error) => return Err(error.into()),
253            }
254        }
255        store.meta = store.read_meta()?;
256        Ok(store)
257    }
258    pub fn existing(home: &Path, id: &str) -> Result<Self, MemoryError> {
259        Self::existing_inner(home, id, true)
260    }
261    pub fn snapshot(home: &Path, id: &str) -> Result<SourceMemory, MemoryError> {
262        let store = Self::existing_inner(home, id, false)?;
263        load_files(&store.files()?, &store.read_meta()?)
264    }
265    fn existing_inner(home: &Path, id: &str, recover: bool) -> Result<Self, MemoryError> {
266        let home = std::path::absolute(home)?;
267        let dir = home.join("memory").join(memory_slug(id, cfg!(windows))?);
268        let mut store = Self {
269            dir,
270            home_directory: std::sync::Arc::new(crate::secure_fs::Directory::open(&home)?),
271            home,
272            meta: MemoryMeta {
273                schema_version: 1,
274                model_id: id.into(),
275                created_at: String::new(),
276                embedding: None,
277                embedding_unsupported: None,
278            },
279        };
280        if recover {
281            store.recover_if_needed()?;
282        } else if store.transaction_dir()?.try_exists()? {
283            return Err(MemoryError(
284                "memory recovery required before preview".into(),
285            ));
286        }
287        store.meta = store.read_meta()?;
288        Ok(store)
289    }
290    fn check_path(&self) -> Result<(), MemoryError> {
291        self.home_directory.check()?;
292        for path in [&self.home, &self.home.join("memory"), &self.dir] {
293            let metadata = fs::symlink_metadata(path)?;
294            if !metadata.is_dir() || metadata.file_type().is_symlink() {
295                return Err(MemoryError("unsafe memory directory".into()));
296            }
297        }
298        #[cfg(unix)]
299        {
300            use std::os::unix::fs::PermissionsExt;
301            if fs::metadata(&self.dir)?.permissions().mode() & 0o077 != 0 {
302                return Err(MemoryError("memory store permissions are too broad".into()));
303            }
304        }
305        if !self
306            .dir
307            .canonicalize()?
308            .starts_with(self.home.join("memory").canonicalize()?)
309        {
310            return Err(MemoryError("memory path escaped root".into()));
311        }
312        Ok(())
313    }
314    pub fn read_meta(&self) -> Result<MemoryMeta, MemoryError> {
315        self.check_path()?;
316        let raw = self.read_at(&self.dir.join("meta.json"), 16384, true)?;
317        let value: serde_json::Value = serde_json::from_str(&raw)?;
318        if value
319            .as_object()
320            .is_some_and(|object| object.values().any(serde_json::Value::is_null))
321        {
322            return Err(MemoryError("null metadata fields".into()));
323        }
324        let meta: MemoryMeta = serde_json::from_value(value)?;
325        meta.validate()?;
326        if meta.model_id != self.meta.model_id {
327            return Err(MemoryError("memory slug collision".into()));
328        }
329        Ok(meta)
330    }
331    pub fn lock(&self) -> Result<LockGuard, MemoryError> {
332        Ok(StateStore::new(Config::from_home(&self.home)?).lock(Duration::from_secs(10))?)
333    }
334    pub fn write_meta(&self, guard: &LockGuard, meta: &MemoryMeta) -> Result<(), MemoryError> {
335        guard.assert_for(&Config::from_home(&self.home)?.lock)?;
336        self.read_meta()?;
337        meta.validate()?;
338        if meta.model_id != self.meta.model_id {
339            return Err(MemoryError("metadata owner changed".into()));
340        }
341        let mut bytes = serde_json::to_vec_pretty(meta)?;
342        bytes.push(b'\n');
343        self.write_at(&self.dir.join("meta.json"), &bytes)
344    }
345    fn relative<'path>(&self, path: &'path Path) -> Result<&'path Path, MemoryError> {
346        path.strip_prefix(&self.home)
347            .map_err(|_| MemoryError("path escaped memory home".into()))
348    }
349    fn mkdir(&self, path: &Path) -> Result<(), MemoryError> {
350        Ok(self.home_directory.ensure_dir(self.relative(path)?)?)
351    }
352    fn write_at(&self, path: &Path, bytes: &[u8]) -> Result<(), MemoryError> {
353        Ok(self
354            .home_directory
355            .write(self.relative(path)?, bytes, false, false)?)
356    }
357    fn read_at(
358        &self,
359        path: &Path,
360        maximum: usize,
361        secret: bool,
362    ) -> Result<String, crate::state::StateError> {
363        let relative = path
364            .strip_prefix(&self.home)
365            .map_err(|_| crate::state::StateError {
366                kind: "invalid",
367                message: "path escaped memory home".into(),
368            })?;
369        let bytes = self
370            .home_directory
371            .read(relative, maximum as u64, secret)
372            .map_err(|error| crate::state::StateError {
373                kind: if error.kind() == std::io::ErrorKind::NotFound {
374                    "absent"
375                } else {
376                    "io"
377                },
378                message: error.to_string(),
379            })?;
380        String::from_utf8(bytes).map_err(|_| crate::state::StateError {
381            kind: "invalid",
382            message: "memory file is not UTF-8".into(),
383        })
384    }
385    fn rename_at(&self, from: impl AsRef<Path>, to: impl AsRef<Path>) -> Result<(), MemoryError> {
386        let directory = self.home_directory.dir()?;
387        directory.rename(
388            self.relative(from.as_ref())?,
389            directory,
390            self.relative(to.as_ref())?,
391        )?;
392        #[cfg(unix)]
393        directory.open(".")?.sync_all()?;
394        Ok(())
395    }
396    fn remove_dir_at(&self, path: impl AsRef<Path>) -> Result<(), MemoryError> {
397        Ok(self
398            .home_directory
399            .dir()?
400            .remove_dir_all(self.relative(path.as_ref())?)?)
401    }
402}
403
404#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
405#[serde(deny_unknown_fields)]
406pub struct Turn {
407    pub role: String,
408    pub content: String,
409    pub ts: String,
410}
411#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
412#[serde(deny_unknown_fields)]
413pub struct Chunk {
414    pub id: String,
415    pub text: String,
416    pub ts: String,
417}
418#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
419#[serde(deny_unknown_fields)]
420pub struct Vector {
421    pub id: String,
422    pub vector: Vec<f64>,
423}
424#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
425#[serde(deny_unknown_fields)]
426pub struct Fact {
427    pub text: String,
428    pub ts: String,
429}
430#[derive(Debug, Clone, Deserialize, Serialize)]
431#[serde(rename_all = "camelCase", deny_unknown_fields)]
432struct Facts {
433    schema_version: u8,
434    facts: Vec<Fact>,
435}
436#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
437#[serde(rename_all = "camelCase", deny_unknown_fields)]
438pub struct SourceMemory {
439    pub turns: Vec<Turn>,
440    #[serde(skip_serializing_if = "Option::is_none")]
441    pub system_prompt: Option<String>,
442    pub facts_text: String,
443    pub facts_present: bool,
444    #[serde(skip_serializing_if = "Option::is_none")]
445    pub embedding: Option<SourceEmbedding>,
446}
447#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
448#[serde(deny_unknown_fields)]
449pub struct SourceEmbedding {
450    pub meta: EmbeddingMeta,
451    pub chunks: Vec<Chunk>,
452    pub vectors: Vec<Vector>,
453}
454pub struct EmbeddingOutput {
455    pub vectors: Vec<Vec<f64>>,
456    pub dimension: usize,
457}
458#[async_trait::async_trait]
459pub trait Embedder: Send + Sync {
460    fn model(&self) -> &str;
461    async fn embed(
462        &self,
463        texts: &[String],
464        cancel: &tokio_util::sync::CancellationToken,
465    ) -> Result<EmbeddingOutput, MemoryError>;
466}
467pub struct CaptureOptions<'capture> {
468    pub timestamp: &'capture str,
469    pub embedder: Option<&'capture dyn Embedder>,
470    pub embedding_unsupported: bool,
471}
472#[derive(Debug, Serialize)]
473#[serde(rename_all = "camelCase")]
474pub struct CaptureResult {
475    pub turns_appended: usize,
476    pub facts_extracted: usize,
477    pub vectors_embedded: usize,
478}
479type Files = std::collections::BTreeMap<String, String>;
480fn jsonl<T: serde::de::DeserializeOwned>(raw: &str) -> Result<Vec<T>, MemoryError> {
481    let mut records = Vec::new();
482    for line in raw.lines().filter(|line| !line.trim().is_empty()) {
483        records.push(serde_json::from_str(line)?);
484        if records.len() > 100000 {
485            return Err(MemoryError("record limit exceeded".into()));
486        }
487    }
488    Ok(records)
489}
490fn encode_lines<T: Serialize>(records: &[T]) -> Result<String, MemoryError> {
491    let mut result = String::new();
492    for record in records {
493        result.push_str(&serde_json::to_string(record)?);
494        result.push('\n');
495    }
496    Ok(result)
497}
498fn pretty<T: Serialize>(value: &T) -> Result<String, MemoryError> {
499    Ok(format!("{}\n", serde_json::to_string_pretty(value)?))
500}
501pub fn extract_facts(text: &str) -> Result<Vec<String>, MemoryError> {
502    let rules = [
503        (
504            r"(?i)(?-u:\b)(?:my name is|call me)\s+([^.,;!?\n]+)",
505            "name = ",
506        ),
507        (
508            r"(?i)(?-u:\b)i (?:live|reside) in\s+([^.,;!?\n]+)",
509            "location = ",
510        ),
511        (
512            r"(?i)(?-u:\b)i work (?:as|at)\s+([^.,;!?\n]+)",
513            "occupation = ",
514        ),
515        (
516            r"(?i)(?-u:\b)i (?:like|love|prefer|enjoy)\s+([^.,;!?\n]+)",
517            "preference = ",
518        ),
519        (r"(?i)(?-u:\b)remember(?: that)?[:\s]+([^.\n]+)", ""),
520    ];
521    let mut result = Vec::new();
522    let mut seen = std::collections::HashSet::new();
523    for (pattern, prefix) in rules {
524        let regex = regex::Regex::new(pattern).map_err(|error| MemoryError(error.to_string()))?;
525        for captures in regex.captures_iter(text) {
526            let Some(value) = captures
527                .get(1)
528                .map(|value| value.as_str().trim())
529                .filter(|value| !value.is_empty())
530            else {
531                continue;
532            };
533            let fact = format!("{prefix}{value}");
534            if seen.insert(fact.to_lowercase()) {
535                result.push(fact);
536            }
537        }
538    }
539    Ok(result)
540}
541fn validate_embeddings(output: &EmbeddingOutput, count: usize) -> Result<(), MemoryError> {
542    if output.dimension == 0
543        || output.dimension > 100000
544        || output.vectors.len() != count
545        || count
546            .checked_mul(output.dimension)
547            .is_none_or(|size| size > 2000000)
548        || output.vectors.iter().any(|vector| {
549            vector.len() != output.dimension || vector.iter().any(|value| !value.is_finite())
550        })
551    {
552        return Err(MemoryError("invalid embedding vectors or dimension".into()));
553    }
554    Ok(())
555}
556impl MemoryStore {
557    fn files(&self) -> Result<Files, MemoryError> {
558        self.read_meta()?;
559        let mut files = Files::new();
560        fn scan(
561            store: &MemoryStore,
562            directory: &cap_std::fs::Dir,
563            relative: &Path,
564            files: &mut Files,
565            total: &mut usize,
566            entries: &mut usize,
567            depth: usize,
568        ) -> Result<(), MemoryError> {
569            if depth > 40 {
570                return Err(MemoryError("memory directory depth exceeds limit".into()));
571            }
572            for entry in directory.entries()? {
573                let entry = entry?;
574                *entries += 1;
575                if *entries > 1024 {
576                    return Err(MemoryError("memory file count exceeds limit".into()));
577                }
578                let kind = entry.file_type()?;
579                let child = relative.join(entry.file_name());
580                if kind.is_dir() {
581                    scan(
582                        store,
583                        &directory.open_dir(entry.file_name())?,
584                        &child,
585                        files,
586                        total,
587                        entries,
588                        depth + 1,
589                    )?;
590                } else if kind.is_file() {
591                    let relative = child
592                        .to_str()
593                        .ok_or_else(|| MemoryError("invalid memory filename encoding".into()))?
594                        .replace('\\', "/");
595                    let maximum = match relative.as_str() {
596                        "meta.json" => 16384,
597                        "facts.json" => 1048576,
598                        "system.md" => 65536,
599                        "embeddings/vectors.jsonl" => 16 * 1024 * 1024,
600                        _ => 8 * 1024 * 1024,
601                    };
602                    let raw = store.read_at(&store.dir.join(&child), maximum, true)?;
603                    *total += raw.len();
604                    if *total > 64 * 1024 * 1024 {
605                        return Err(MemoryError("memory store exceeds limit".into()));
606                    }
607                    files.insert(relative, raw);
608                } else {
609                    return Err(MemoryError("symlink or special memory file".into()));
610                }
611            }
612            Ok(())
613        }
614        let directory = self.home_directory.subdir(self.relative(&self.dir)?)?;
615        scan(
616            self,
617            &directory,
618            Path::new(""),
619            &mut files,
620            &mut 0,
621            &mut 0,
622            0,
623        )?;
624        Ok(files)
625    }
626    pub fn load(&self) -> Result<SourceMemory, MemoryError> {
627        self.recover_if_needed()?;
628        let files = self.files()?;
629        load_files(&files, &self.read_meta()?)
630    }
631    fn transaction_dir(&self) -> Result<PathBuf, MemoryError> {
632        Ok(self.home.join(".staging").join(format!(
633            "memory-{}",
634            memory_slug(&self.meta.model_id, cfg!(windows))?
635        )))
636    }
637    pub fn recover(&self, guard: &LockGuard) -> Result<(), MemoryError> {
638        guard.assert_for(&Config::from_home(&self.home)?.lock)?;
639        let transaction = self.transaction_dir()?;
640        if !transaction.try_exists()? {
641            return Ok(());
642        }
643        self.home_directory.subdir(self.relative(&transaction)?)?;
644        let marker = match self.read_at(&transaction.join("phase"), 32, true) {
645            Ok(marker) => marker,
646            Err(error) if error.kind == "absent" => "staging".into(),
647            Err(error) => return Err(error.into()),
648        };
649        let previous = transaction.join("previous");
650        let next = transaction.join("next");
651        if marker.trim() == "moving" {
652            let target_id = self.read_at(&transaction.join("target"), 8192, true)?;
653            if memory_slug(&target_id, cfg!(windows))?
654                == memory_slug(&self.meta.model_id, cfg!(windows))?
655            {
656                return Err(MemoryError("recursive memory recovery".into()));
657            }
658            let target = Self {
659                home_directory: self.home_directory.clone(),
660                dir: self
661                    .home
662                    .join("memory")
663                    .join(memory_slug(&target_id, cfg!(windows))?),
664                home: self.home.clone(),
665                meta: MemoryMeta {
666                    model_id: target_id,
667                    ..self.meta.clone()
668                },
669            };
670            if target.transaction_dir()?.try_exists()? {
671                let phase = self.read_at(&target.transaction_dir()?.join("phase"), 32, true)?;
672                if !["prepared", "committed"].contains(&phase.trim()) {
673                    return Err(MemoryError("invalid move target journal".into()));
674                }
675                target.recover(guard)?;
676            }
677            if transaction.try_exists()? {
678                self.remove_dir_at(transaction)?;
679            }
680            return Ok(());
681        }
682        if marker.trim() == "staging" {
683            if previous.try_exists()? {
684                return Err(MemoryError("ambiguous staging journal".into()));
685            }
686            self.read_meta()?;
687            let quarantine = self
688                .home
689                .join(".staging")
690                .join(format!("memory-recovered-{}", uuid::Uuid::new_v4()));
691            self.rename_at(transaction, quarantine)?;
692            return Ok(());
693        }
694        if marker.trim() == "committed" {
695            self.read_meta()?;
696            self.recover_move(guard, false)?;
697            self.remove_dir_at(transaction)?;
698            return Ok(());
699        }
700        if marker.trim() != "prepared" {
701            return Err(MemoryError("invalid memory recovery journal".into()));
702        }
703        if previous.try_exists()? {
704            self.home_directory.subdir(self.relative(&previous)?)?;
705            if self.dir.try_exists()? {
706                self.check_path()?;
707                if next.try_exists()? {
708                    return Err(MemoryError(
709                        "ambiguous memory recovery; preserve all generations".into(),
710                    ));
711                }
712                self.rename_at(&self.dir, &next)?;
713            }
714            self.rename_at(&previous, &self.dir)?;
715        }
716        self.read_meta()?;
717        self.recover_move(guard, true)?;
718        if next.try_exists()? {
719            let quarantine = self
720                .home
721                .join(".staging")
722                .join(format!("memory-recovered-{}", uuid::Uuid::new_v4()));
723            self.rename_at(&next, quarantine)?;
724        }
725        self.remove_dir_at(transaction)?;
726        Ok(())
727    }
728    fn recover_if_needed(&self) -> Result<(), MemoryError> {
729        if self.transaction_dir()?.try_exists()? {
730            let guard = self.lock()?;
731            self.recover(&guard)?;
732            guard.release()?;
733        }
734        Ok(())
735    }
736    fn publish(
737        &self,
738        guard: &LockGuard,
739        expected: &Files,
740        next: &Files,
741    ) -> Result<(), MemoryError> {
742        self.publish_pending(guard, expected, next)?;
743        self.commit_pending(guard)
744    }
745    fn publish_pending(
746        &self,
747        guard: &LockGuard,
748        expected: &Files,
749        next: &Files,
750    ) -> Result<(), MemoryError> {
751        guard.assert_for(&Config::from_home(&self.home)?.lock)?;
752        self.recover(guard)?;
753        if &self.files()? != expected {
754            return Err(MemoryError("memory changed during preparation".into()));
755        }
756        let transaction = self.transaction_dir()?;
757        self.mkdir(&self.home.join(".staging"))?;
758        if transaction.try_exists()? {
759            return Err(MemoryError("memory transaction already exists".into()));
760        }
761        self.mkdir(&transaction)?;
762        self.write_at(&transaction.join("phase"), b"staging\n")?;
763        let staged = transaction.join("next");
764        self.mkdir(&staged)?;
765        let staged_result = (|| {
766            for (name, raw) in next {
767                if !crate::acquire::safe_file(name) {
768                    return Err(MemoryError("unsafe memory artifact name".into()));
769                }
770                let path = staged.join(name);
771                self.mkdir(
772                    path.parent()
773                        .ok_or_else(|| MemoryError("missing memory parent".into()))?,
774                )?;
775                self.write_at(&path, raw.as_bytes())?;
776            }
777            self.write_at(&transaction.join("phase"), b"prepared\n")?;
778            Ok::<_, MemoryError>(())
779        })();
780        if let Err(error) = staged_result {
781            self.remove_dir_at(&transaction)?;
782            return Err(error);
783        }
784        let commit = (|| {
785            guard.assert_owned()?;
786            if &self.files()? != expected {
787                return Err(MemoryError("memory changed before publication".into()));
788            }
789            self.rename_at(&self.dir, transaction.join("previous"))?;
790            self.rename_at(&staged, &self.dir)?;
791            if self.files()? != *next {
792                return Err(MemoryError("published memory verification failed".into()));
793            }
794            Ok::<_, MemoryError>(())
795        })();
796        if let Err(error) = commit {
797            self.recover(guard)?;
798            return Err(error);
799        }
800        Ok(())
801    }
802    fn commit_pending(&self, guard: &LockGuard) -> Result<(), MemoryError> {
803        guard.assert_for(&Config::from_home(&self.home)?.lock)?;
804        self.write_at(&self.transaction_dir()?.join("phase"), b"committed\n")?;
805        self.recover(guard)
806    }
807    fn prepare_move(&self, source: &MemoryStore, guard: &LockGuard) -> Result<(), MemoryError> {
808        guard.assert_for(&Config::from_home(&self.home)?.lock)?;
809        if source.home != self.home {
810            return Err(MemoryError("move crosses memory homes".into()));
811        }
812        source.check_path()?;
813        let recovery = format!("memory-moved-{}", uuid::Uuid::new_v4());
814        self.write_at(
815            &self.transaction_dir()?.join("move.json"),
816            &serde_json::to_vec(&MoveJournal {
817                source: source.meta.model_id.clone(),
818                recovery: recovery.clone(),
819            })?,
820        )?;
821        let source_transaction = source.transaction_dir()?;
822        if source_transaction.try_exists()? {
823            return Err(MemoryError("source transaction exists".into()));
824        }
825        self.mkdir(&source_transaction)?;
826        self.write_at(
827            &source_transaction.join("target"),
828            self.meta.model_id.as_bytes(),
829        )?;
830        self.write_at(&source_transaction.join("phase"), b"moving\n")?;
831        self.rename_at(&source.dir, self.home.join(".staging").join(recovery))?;
832        Ok(())
833    }
834    fn recover_move(&self, guard: &LockGuard, rollback: bool) -> Result<(), MemoryError> {
835        guard.assert_for(&Config::from_home(&self.home)?.lock)?;
836        let raw = match self.read_at(&self.transaction_dir()?.join("move.json"), 16384, true) {
837            Ok(raw) => raw,
838            Err(error) if error.kind == "absent" => return Ok(()),
839            Err(error) => return Err(error.into()),
840        };
841        let journal: MoveJournal = serde_json::from_str(&raw)?;
842        let suffix = journal
843            .recovery
844            .strip_prefix("memory-moved-")
845            .ok_or_else(|| MemoryError("invalid move recovery name".into()))?;
846        if uuid::Uuid::parse_str(suffix).is_err() {
847            return Err(MemoryError("invalid move recovery id".into()));
848        }
849        let slug = memory_slug(&journal.source, cfg!(windows))?;
850        if slug == memory_slug(&self.meta.model_id, cfg!(windows))? {
851            return Err(MemoryError("invalid move source".into()));
852        }
853        let source = self.home.join("memory").join(&slug);
854        let recovery = self.home.join(".staging").join(&journal.recovery);
855        if rollback && recovery.try_exists()? {
856            self.home_directory.subdir(self.relative(&recovery)?)?;
857            let meta: MemoryMeta =
858                serde_json::from_str(&self.read_at(&recovery.join("meta.json"), 16384, true)?)?;
859            meta.validate()?;
860            if meta.model_id != journal.source || source.try_exists()? {
861                return Err(MemoryError(
862                    "ambiguous source recovery; bytes retained".into(),
863                ));
864            }
865            self.rename_at(recovery, source)?;
866        }
867        let source_transaction = self.home.join(".staging").join(format!("memory-{slug}"));
868        if source_transaction.try_exists()? {
869            self.home_directory
870                .subdir(self.relative(&source_transaction)?)?;
871            let target = self.read_at(&source_transaction.join("target"), 8192, true)?;
872            if target != self.meta.model_id {
873                return Err(MemoryError("move recovery target changed".into()));
874            }
875            self.remove_dir_at(source_transaction)?;
876        }
877        Ok(())
878    }
879    pub async fn capture(
880        &self,
881        user: &str,
882        assistant: &str,
883        options: CaptureOptions<'_>,
884        cancel: &tokio_util::sync::CancellationToken,
885    ) -> Result<CaptureResult, MemoryError> {
886        if user.len() + assistant.len() > 8 * 1024 * 1024
887            || options.timestamp.is_empty()
888            || options.timestamp.len() > 1024
889        {
890            return Err(MemoryError("invalid capture input".into()));
891        }
892        if cancel.is_cancelled() {
893            return Err(MemoryError("cancelled".into()));
894        }
895        self.recover_if_needed()?;
896        let expected = self.files()?;
897        let mut source = load_files(&expected, &self.read_meta()?)?;
898        let mut meta = self.read_meta()?;
899        let texts = [
900            rigspark_core::reports::strip_control(user),
901            rigspark_core::reports::strip_control(assistant),
902        ];
903        let chunks: Vec<_> = texts
904            .iter()
905            .filter(|text| !text.is_empty())
906            .cloned()
907            .collect();
908        let embedding = if !options.embedding_unsupported && !chunks.is_empty() {
909            if let Some(embedder) = options.embedder {
910                let result = tokio::select! {biased;_=cancel.cancelled()=>return Err(MemoryError("cancelled".into())),result=tokio::time::timeout(Duration::from_secs(120),embedder.embed(&chunks,cancel))=>result.map_err(|_|MemoryError("embedding timed out".into()))??};
911                validate_embeddings(&result, chunks.len())?;
912                let space = EmbeddingMeta {
913                    model: embedder.model().into(),
914                    dimension: result.dimension,
915                };
916                if space.model.is_empty()
917                    || meta
918                        .embedding
919                        .as_ref()
920                        .is_some_and(|existing| existing != &space)
921                {
922                    return Err(MemoryError("embedding space mismatch".into()));
923                }
924                Some((space, result))
925            } else {
926                None
927            }
928        } else {
929            None
930        };
931        let before = source.turns.len();
932        for (role, content) in ["user", "assistant"].into_iter().zip(&texts) {
933            if !content.is_empty() {
934                source.turns.push(Turn {
935                    role: role.into(),
936                    content: content.clone(),
937                    ts: options.timestamp.into(),
938                });
939            }
940        }
941        let mut facts = if source.facts_present {
942            serde_json::from_str::<Facts>(&source.facts_text)?
943        } else {
944            Facts {
945                schema_version: 1,
946                facts: Vec::new(),
947            }
948        };
949        validate_facts(&facts)?;
950        let mut known: std::collections::HashSet<_> = facts
951            .facts
952            .iter()
953            .map(|fact| fact.text.to_lowercase())
954            .collect();
955        let mut added = 0;
956        for text in extract_facts(&texts[0])? {
957            if known.insert(text.to_lowercase()) {
958                facts.facts.push(Fact {
959                    text,
960                    ts: options.timestamp.into(),
961                });
962                added += 1;
963            }
964        }
965        let mut next = expected.clone();
966        next.insert("conversation.jsonl".into(), encode_lines(&source.turns)?);
967        if added > 0 || !source.facts_present {
968            next.insert("facts.json".into(), pretty(&facts)?);
969        }
970        let mut embedded = 0;
971        if options.embedding_unsupported {
972            meta.embedding = None;
973            meta.embedding_unsupported = Some(true);
974        } else if let Some((space, output)) = embedding {
975            let mut index = source.embedding.unwrap_or(SourceEmbedding {
976                meta: space.clone(),
977                chunks: Vec::new(),
978                vectors: Vec::new(),
979            });
980            for (text, vector) in chunks.into_iter().zip(output.vectors) {
981                let id = uuid::Uuid::new_v4().to_string();
982                index.chunks.push(Chunk {
983                    id: id.clone(),
984                    text,
985                    ts: options.timestamp.into(),
986                });
987                index.vectors.push(Vector { id, vector });
988                embedded += 1;
989            }
990            meta.embedding = Some(space);
991            meta.embedding_unsupported = None;
992            next.insert(
993                "embeddings/chunks.jsonl".into(),
994                encode_lines(&index.chunks)?,
995            );
996            next.insert(
997                "embeddings/vectors.jsonl".into(),
998                encode_lines(&index.vectors)?,
999            );
1000        }
1001        meta.validate()?;
1002        next.insert("meta.json".into(), pretty(&meta)?);
1003        load_files(&next, &meta)?;
1004        let guard = self.lock()?;
1005        if cancel.is_cancelled() {
1006            return Err(MemoryError("cancelled".into()));
1007        }
1008        self.publish(&guard, &expected, &next)?;
1009        guard.release()?;
1010        Ok(CaptureResult {
1011            turns_appended: source.turns.len() - before,
1012            facts_extracted: added,
1013            vectors_embedded: embedded,
1014        })
1015    }
1016}
1017fn validate_facts(facts: &Facts) -> Result<(), MemoryError> {
1018    if facts.schema_version != 1
1019        || facts.facts.len() > 100000
1020        || facts
1021            .facts
1022            .iter()
1023            .any(|fact| fact.text.is_empty() || fact.ts.is_empty())
1024    {
1025        return Err(MemoryError("invalid facts".into()));
1026    }
1027    Ok(())
1028}
1029fn load_files(files: &Files, meta: &MemoryMeta) -> Result<SourceMemory, MemoryError> {
1030    let turns: Vec<Turn> = jsonl(
1031        files
1032            .get("conversation.jsonl")
1033            .map(String::as_str)
1034            .unwrap_or(""),
1035    )?;
1036    if turns
1037        .iter()
1038        .any(|turn| !["user", "assistant", "system"].contains(&turn.role.as_str()))
1039    {
1040        return Err(MemoryError("invalid stored role".into()));
1041    }
1042    let embedding = if let Some(space) = &meta.embedding {
1043        let chunks: Vec<Chunk> = jsonl(
1044            files
1045                .get("embeddings/chunks.jsonl")
1046                .map(String::as_str)
1047                .unwrap_or(""),
1048        )?;
1049        let vectors: Vec<Vector> = jsonl(
1050            files
1051                .get("embeddings/vectors.jsonl")
1052                .map(String::as_str)
1053                .unwrap_or(""),
1054        )?;
1055        let ids: std::collections::HashSet<_> = chunks.iter().map(|chunk| &chunk.id).collect();
1056        let vector_ids: std::collections::HashSet<_> =
1057            vectors.iter().map(|vector| &vector.id).collect();
1058        if ids.len() != chunks.len()
1059            || vector_ids.len() != vectors.len()
1060            || ids != vector_ids
1061            || vectors.iter().any(|vector| {
1062                vector.vector.len() != space.dimension
1063                    || vector.vector.iter().any(|value| !value.is_finite())
1064            })
1065        {
1066            return Err(MemoryError("inconsistent embedding index".into()));
1067        }
1068        Some(SourceEmbedding {
1069            meta: space.clone(),
1070            chunks,
1071            vectors,
1072        })
1073    } else {
1074        None
1075    };
1076    let facts_text = files.get("facts.json").cloned().unwrap_or_default();
1077    let facts_present = files.contains_key("facts.json");
1078    Ok(SourceMemory {
1079        turns,
1080        embedding,
1081        system_prompt: files.get("system.md").cloned(),
1082        facts_text,
1083        facts_present,
1084    })
1085}
1086
1087#[async_trait::async_trait]
1088pub trait Summarizer: Send + Sync {
1089    async fn summarize(
1090        &self,
1091        turns: &[Turn],
1092        cancel: &tokio_util::sync::CancellationToken,
1093    ) -> Result<String, MemoryError>;
1094}
1095pub struct MigrationOptions<'migration> {
1096    pub context: u32,
1097    pub embedder: Option<&'migration dyn Embedder>,
1098    pub target_dimension: Option<usize>,
1099    pub summarizer: Option<&'migration dyn Summarizer>,
1100    pub embedding_unsupported: bool,
1101}
1102#[derive(Debug, Clone, Serialize)]
1103#[serde(rename_all = "camelCase")]
1104pub struct MigrationSummary {
1105    pub turns_carried: usize,
1106    pub turns_summarized: usize,
1107    pub vectors_reembedded: usize,
1108    pub strategy: String,
1109    pub embedding_strategy: String,
1110}
1111pub struct MigrationPlan {
1112    pub source: SourceMemory,
1113    pub embedding_unsupported: bool,
1114    pub summary: MigrationSummary,
1115}
1116fn token_count(text: &str) -> usize {
1117    text.encode_utf16().count().div_ceil(4)
1118}
1119fn truncate_utf16(text: &str, maximum: usize) -> String {
1120    let mut length = 0;
1121    text.chars()
1122        .take_while(|character| {
1123            length += character.len_utf16();
1124            length <= maximum
1125        })
1126        .collect()
1127}
1128pub async fn plan_migration(
1129    source: &SourceMemory,
1130    options: MigrationOptions<'_>,
1131    cancel: &tokio_util::sync::CancellationToken,
1132) -> Result<MigrationPlan, MemoryError> {
1133    if options.context == 0 || options.context > 10000000 || source.turns.len() > 100000 {
1134        return Err(MemoryError("invalid migration budget or history".into()));
1135    }
1136    if cancel.is_cancelled() {
1137        return Err(MemoryError("cancelled".into()));
1138    }
1139    let mut output = source.clone();
1140    let mut summary = MigrationSummary {
1141        turns_carried: source.turns.len(),
1142        turns_summarized: 0,
1143        vectors_reembedded: 0,
1144        strategy: "none".into(),
1145        embedding_strategy: "none".into(),
1146    };
1147    let reserved = token_count(source.system_prompt.as_deref().unwrap_or(""))
1148        + token_count(&source.facts_text);
1149    let turn_cost = source
1150        .turns
1151        .iter()
1152        .map(|turn| token_count(&turn.content))
1153        .sum::<usize>();
1154    if reserved + turn_cost > options.context as usize {
1155        let available = (options.context as usize).saturating_sub(reserved + 256);
1156        let mut running = 0;
1157        let mut kept = 0;
1158        for turn in source.turns.iter().rev() {
1159            let cost = token_count(&turn.content);
1160            if running + cost > available {
1161                break;
1162            }
1163            running += cost;
1164            kept += 1;
1165        }
1166        let overflow = source.turns.len() - kept;
1167        if overflow > 0 {
1168            let content = if let Some(summarizer) = options.summarizer {
1169                let text = tokio::select! {biased;_=cancel.cancelled()=>return Err(MemoryError("cancelled".into())),result=tokio::time::timeout(Duration::from_secs(120),summarizer.summarize(&source.turns[..overflow],cancel))=>result.map_err(|_|MemoryError("summarization timed out".into()))??};
1170                summary.strategy = "summarize".into();
1171                truncate_utf16(
1172                    &format!(
1173                        "Summary of prior conversation: {}",
1174                        rigspark_core::reports::strip_control(&text)
1175                    ),
1176                    1024,
1177                )
1178            } else {
1179                summary.strategy = "truncate".into();
1180                format!("[{overflow} earlier turns omitted during migration]")
1181            };
1182            output.turns = vec![Turn {
1183                role: "system".into(),
1184                content,
1185                ts: source.turns[0].ts.clone(),
1186            }];
1187            output.turns.extend_from_slice(&source.turns[overflow..]);
1188            summary.turns_carried = kept;
1189            summary.turns_summarized = overflow;
1190        }
1191    }
1192    if options.embedding_unsupported {
1193        output.embedding = None;
1194    } else if let Some(index) = &source.embedding {
1195        if !index.chunks.is_empty() {
1196            if let Some(embedder) = options.embedder.filter(|embedder| {
1197                embedder.model() != index.meta.model
1198                    || options.target_dimension != Some(index.meta.dimension)
1199            }) {
1200                let dimension = options
1201                    .target_dimension
1202                    .filter(|dimension| *dimension > 0 && *dimension <= 100000)
1203                    .ok_or_else(|| MemoryError("target embedding dimension required".into()))?;
1204                let texts: Vec<_> = index
1205                    .chunks
1206                    .iter()
1207                    .map(|chunk| chunk.text.clone())
1208                    .collect();
1209                let output_vectors = tokio::select! {biased;_=cancel.cancelled()=>return Err(MemoryError("cancelled".into())),result=tokio::time::timeout(Duration::from_secs(120),embedder.embed(&texts,cancel))=>result.map_err(|_|MemoryError("reembedding timed out".into()))??};
1210                validate_embeddings(&output_vectors, texts.len())?;
1211                if output_vectors.dimension != dimension {
1212                    return Err(MemoryError("reembedding dimension mismatch".into()));
1213                }
1214                let vectors = index
1215                    .chunks
1216                    .iter()
1217                    .zip(output_vectors.vectors)
1218                    .map(|(chunk, vector)| Vector {
1219                        id: chunk.id.clone(),
1220                        vector,
1221                    })
1222                    .collect();
1223                output.embedding = Some(SourceEmbedding {
1224                    meta: EmbeddingMeta {
1225                        model: embedder.model().into(),
1226                        dimension,
1227                    },
1228                    chunks: index.chunks.clone(),
1229                    vectors,
1230                });
1231                summary.embedding_strategy = "reembed".into();
1232                summary.vectors_reembedded = texts.len();
1233            } else {
1234                summary.embedding_strategy = "reuse".into();
1235            }
1236        } else {
1237            output.embedding = None;
1238        }
1239    }
1240    Ok(MigrationPlan {
1241        source: output,
1242        embedding_unsupported: options.embedding_unsupported,
1243        summary,
1244    })
1245}
1246pub struct PreparedMigration {
1247    source_dir: PathBuf,
1248    source_id: String,
1249    source_files: Files,
1250    target_id: String,
1251    prior_target: Option<Files>,
1252    plan: MigrationPlan,
1253}
1254impl PreparedMigration {
1255    pub fn summary(&self) -> &MigrationSummary {
1256        &self.plan.summary
1257    }
1258}
1259impl MemoryStore {
1260    pub async fn migrate(
1261        &self,
1262        target_id: &str,
1263        created_at: &str,
1264        options: MigrationOptions<'_>,
1265        move_source: bool,
1266        cancel: &tokio_util::sync::CancellationToken,
1267    ) -> Result<MemoryStore, MemoryError> {
1268        let prepared = self.prepare_migration(target_id, options, cancel).await?;
1269        self.commit_migration(prepared, created_at, move_source, cancel)
1270    }
1271    pub async fn prepare_migration(
1272        &self,
1273        target_id: &str,
1274        options: MigrationOptions<'_>,
1275        cancel: &tokio_util::sync::CancellationToken,
1276    ) -> Result<PreparedMigration, MemoryError> {
1277        if memory_slug(target_id, cfg!(windows))?
1278            == memory_slug(&self.meta.model_id, cfg!(windows))?
1279        {
1280            return Err(MemoryError("source and target stores must differ".into()));
1281        }
1282        self.recover_if_needed()?;
1283        let source_files = self.files()?;
1284        let source = load_files(&source_files, &self.read_meta()?)?;
1285        let target_path = self
1286            .home
1287            .join("memory")
1288            .join(memory_slug(target_id, cfg!(windows))?);
1289        let target_probe = Self {
1290            home_directory: self.home_directory.clone(),
1291            dir: target_path.clone(),
1292            home: self.home.clone(),
1293            meta: MemoryMeta {
1294                model_id: target_id.into(),
1295                ..self.meta.clone()
1296            },
1297        };
1298        target_probe.recover_if_needed()?;
1299        let prior_target = if target_path.try_exists()? {
1300            Some(Self::existing(&self.home, target_id)?.files()?)
1301        } else {
1302            None
1303        };
1304        let plan = plan_migration(&source, options, cancel).await?;
1305        Ok(PreparedMigration {
1306            source_dir: self.dir.clone(),
1307            source_id: self.meta.model_id.clone(),
1308            source_files,
1309            target_id: target_id.into(),
1310            prior_target,
1311            plan,
1312        })
1313    }
1314    pub fn commit_migration(
1315        &self,
1316        prepared: PreparedMigration,
1317        created_at: &str,
1318        move_source: bool,
1319        cancel: &tokio_util::sync::CancellationToken,
1320    ) -> Result<MemoryStore, MemoryError> {
1321        self.commit_migration_with(prepared, created_at, move_source, cancel, || Ok(()))
1322    }
1323    pub fn commit_migration_with(
1324        &self,
1325        prepared: PreparedMigration,
1326        created_at: &str,
1327        move_source: bool,
1328        cancel: &tokio_util::sync::CancellationToken,
1329        check: impl FnOnce() -> Result<(), MemoryError>,
1330    ) -> Result<MemoryStore, MemoryError> {
1331        if prepared.source_dir != self.dir || prepared.source_id != self.meta.model_id {
1332            return Err(MemoryError(
1333                "prepared migration belongs to another source".into(),
1334            ));
1335        }
1336        let PreparedMigration {
1337            source_files,
1338            target_id,
1339            prior_target,
1340            plan,
1341            ..
1342        } = prepared;
1343        let target_id = target_id.as_str();
1344        let target_path = self
1345            .home
1346            .join("memory")
1347            .join(memory_slug(target_id, cfg!(windows))?);
1348        let target_probe = Self {
1349            dir: target_path.clone(),
1350            home: self.home.clone(),
1351            home_directory: self.home_directory.clone(),
1352            meta: MemoryMeta {
1353                model_id: target_id.into(),
1354                ..self.meta.clone()
1355            },
1356        };
1357        let guard = self.lock()?;
1358        check()?;
1359        if cancel.is_cancelled() {
1360            return Err(MemoryError("cancelled".into()));
1361        }
1362        if self.files()? != source_files {
1363            return Err(MemoryError(
1364                "source memory changed during preparation".into(),
1365            ));
1366        }
1367        if target_probe.transaction_dir()?.try_exists()? {
1368            return Err(MemoryError(
1369                "target transaction changed during preparation".into(),
1370            ));
1371        }
1372        let current_target = if target_path.try_exists()? {
1373            Some(Self::existing(&self.home, target_id)?.files()?)
1374        } else {
1375            None
1376        };
1377        if current_target != prior_target {
1378            return Err(MemoryError(
1379                "target memory changed during preparation".into(),
1380            ));
1381        }
1382        let mut target = target_probe;
1383        target.meta.created_at = created_at.into();
1384        target.meta.embedding = None;
1385        target.meta.embedding_unsupported = None;
1386        let target = target.initialize()?;
1387        let expected = target.files()?;
1388        let mut meta = target.read_meta()?;
1389        meta.embedding = plan
1390            .source
1391            .embedding
1392            .as_ref()
1393            .map(|index| index.meta.clone());
1394        meta.embedding_unsupported = plan.embedding_unsupported.then_some(true);
1395        let mut next = source_files.clone();
1396        next.insert("meta.json".into(), pretty(&meta)?);
1397        next.insert(
1398            "conversation.jsonl".into(),
1399            encode_lines(&plan.source.turns)?,
1400        );
1401        if let Some(persona) = &plan.source.system_prompt {
1402            next.insert("system.md".into(), persona.clone());
1403        } else {
1404            next.remove("system.md");
1405        }
1406        if plan.source.facts_present {
1407            next.insert("facts.json".into(), plan.source.facts_text.clone());
1408        } else {
1409            next.remove("facts.json");
1410        }
1411        if let Some(index) = &plan.source.embedding {
1412            next.insert(
1413                "embeddings/chunks.jsonl".into(),
1414                encode_lines(&index.chunks)?,
1415            );
1416            next.insert(
1417                "embeddings/vectors.jsonl".into(),
1418                encode_lines(&index.vectors)?,
1419            );
1420        } else {
1421            next.remove("embeddings/chunks.jsonl");
1422            next.remove("embeddings/vectors.jsonl");
1423        }
1424        load_files(&next, &meta)?;
1425        if let Err(error) = target.publish_pending(&guard, &expected, &next) {
1426            if prior_target.is_none() && target.dir.try_exists()? && target.files()? == expected {
1427                self.remove_dir_at(&target.dir)?;
1428            }
1429            return Err(error);
1430        }
1431        let finish = (|| {
1432            if move_source {
1433                if self.files()? != source_files {
1434                    return Err(MemoryError("source changed before retirement".into()));
1435                }
1436                target.prepare_move(self, &guard)?;
1437            }
1438            target.commit_pending(&guard)
1439        })();
1440        if let Err(error) = finish {
1441            target.recover(&guard)?;
1442            if prior_target.is_none() && target.files()? == expected {
1443                self.remove_dir_at(&target.dir)?;
1444            }
1445            return Err(error);
1446        }
1447        guard.release()?;
1448        Self::existing(&self.home, target_id)
1449    }
1450}