memfork 0.3.0

Git for agent memory, shared by every AI tool on your machine: fork, merge, rewind and hand off work between agents over MCP.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
//! Single-writer enforcement, and where to find a running daemon.
//!
//! Two files, deliberately:
//!
//! - **`memfork.lock`** is empty and stays empty. Its only job is to be held
//!   under an advisory exclusive lock for as long as a process owns the data
//!   directory.
//! - **`memfork.endpoint`** is ordinary JSON, never locked, holding the owning
//!   process's id and — once the daemon exists — its port and token.
//!
//! They are separate because on Windows an exclusive `LockFileEx` range
//! blocks *reads* as well as writes, so a client that needed the token would
//! be unable to read the very file proving the daemon was alive. Keeping the
//! lock on an empty file and the facts beside it avoids that entirely, and
//! costs nothing on Unix.
//!
//! Locking uses `std::fs::File::try_lock`, so there is no locking crate to
//! depend on. On Windows it is `LockFileEx`, which blocks *reads* of the
//! locked range as well as writes — verified, not assumed: reading a locked
//! file there fails with "another process has locked a portion of the file".
//! That is precisely why the token is not kept in the locked file.
//!
//! **Staleness needs no heuristics.** Every operating system releases an
//! advisory lock when the holding process dies, however it dies. So acquiring
//! the lock *is* proof that the previous owner is gone — no process-liveness
//! guessing, no start-time comparison, and the same answer on all three
//! platforms. An endpoint file left behind by a dead owner is then deleted.
//!
//! **Asking is taking, for a moment.** [`owner`] finds out whether anyone
//! holds the lock by trying to take it, and lets go at once. A process that
//! asks while a daemon is starting can therefore hold the lock at the very
//! instant the daemon tries for it. So [`DirLock::acquire`] keeps trying for
//! up to [`PROBE_GRACE`] before it concludes the directory is owned — a
//! question holds the lock for microseconds, an owner for as long as it runs,
//! and an owner publishes an endpoint file, which ends the wait at once — and
//! [`owner`] removes a stale endpoint file only while it still holds the
//! lock, so it can never remove the file of a daemon that started a moment
//! after it let go.
//!
//! On permissions, the two platforms are not equal and this does not pretend
//! otherwise. On Unix the endpoint file is created `0600`. On Windows setting
//! an explicit ACL needs `unsafe` Win32 calls, which this phase does not
//! allow, so the file relies on living inside `%LOCALAPPDATA%`, which is
//! already user-scoped by default. That is weaker: an administrator, or
//! anything running as the user, can read it.

use std::fs::{File, OpenOptions};
use std::io::Write;
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};

use serde::{Deserialize, Serialize};

/// The file that is locked, and holds nothing.
pub const LOCK_FILE: &str = "memfork.lock";

/// The file that holds the facts, and is never locked.
pub const ENDPOINT_FILE: &str = "memfork.endpoint";

/// How long [`DirLock::acquire`] keeps trying a lock that is held, in case
/// the holder is only [`owner`] asking, before saying the directory is owned.
pub const PROBE_GRACE: Duration = Duration::from_secs(1);

/// What a running MemFork process publishes about itself.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Endpoint {
    /// Format version of this file.
    pub version: u32,
    /// The process that owns the data directory.
    pub pid: u32,
    /// The daemon's port, once there is a daemon to have one.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub port: Option<u16>,
    /// The bearer token a client must present, alongside the port.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub token: Option<String>,
    /// A second token the daemon accepts only on what reads: the Brain's
    /// routes and the event stream. The page holds this one and never the
    /// other, so nothing it could do reaches a route that writes.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub read_token: Option<String>,
    /// When the owner started, in seconds since the epoch. Informational.
    pub started_unix: u64,
    /// Which build of MemFork is running.
    ///
    /// So a client can refuse to talk to a daemon from another version rather
    /// than assume the two agree about the formats they write. Optional only
    /// because an endpoint file written before this field existed has none,
    /// and that absence is itself a mismatch.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub memfork_version: Option<String>,
    /// Which build of the Brain's page the daemon serves: the page build of
    /// the binary it was started from (`brain::page_build`). Two binaries of
    /// one version differ in it whenever their page differs, which a version
    /// string cannot say; `memfork brain` holds it against its own and
    /// `memfork doctor` shows both. `None` from a build without the Brain,
    /// or from an endpoint file written before the field existed.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub page_build: Option<String>,
}

/// The version this build writes.
pub const ENDPOINT_VERSION: u32 = 1;

impl Endpoint {
    /// An endpoint for this process, with no daemon details yet.
    pub fn for_this_process() -> Self {
        Endpoint {
            version: ENDPOINT_VERSION,
            pid: std::process::id(),
            port: None,
            token: None,
            read_token: None,
            started_unix: std::time::SystemTime::now()
                .duration_since(std::time::UNIX_EPOCH)
                .map(|d| d.as_secs())
                .unwrap_or(0),
            memfork_version: Some(crate::VERSION.to_owned()),
            page_build: this_page_build(),
        }
    }
}

/// The page build this binary carries, or `None` when it carries no page.
fn this_page_build() -> Option<String> {
    #[cfg(feature = "brain")]
    {
        Some(crate::brain::page_build().to_owned())
    }
    #[cfg(not(feature = "brain"))]
    {
        None
    }
}

/// Why the data directory could not be taken.
#[derive(Debug, thiserror::Error)]
pub enum LockError {
    /// Another running process holds it.
    #[error(
        "this data directory is in use by process {pid}{}.\n\
         Only one process can write to a MemFork data directory at a time.",
        .hint.as_deref().unwrap_or("")
    )]
    InUse {
        /// The owning process id.
        pid: u32,
        /// Anything else worth saying about the owner.
        hint: Option<String>,
    },
    /// Something went wrong with the files themselves.
    #[error("{context}: {source}")]
    Io {
        /// What was being attempted.
        context: String,
        /// The underlying failure.
        #[source]
        source: std::io::Error,
    },
}

fn io(context: impl Into<String>) -> impl FnOnce(std::io::Error) -> LockError {
    let context = context.into();
    move |source| LockError::Io { context, source }
}

/// Ownership of a data directory, held until dropped.
#[derive(Debug)]
pub struct DirLock {
    dir: PathBuf,
    /// Holding the handle is what holds the lock; the operating system
    /// releases it when this closes, including when the process dies.
    file: File,
}

impl DirLock {
    /// Take exclusive ownership of a data directory.
    ///
    /// Fails with [`LockError::InUse`], naming the owner, if another running
    /// process has it.
    pub fn acquire(dir: &Path) -> Result<Self, LockError> {
        std::fs::create_dir_all(dir).map_err(io(format!("cannot create {}", dir.display())))?;
        let path = dir.join(LOCK_FILE);
        let file = OpenOptions::new()
            .read(true)
            .write(true)
            .create(true)
            .truncate(false)
            .open(&path)
            .map_err(io(format!("cannot open {}", path.display())))?;

        // A held lock may only be somebody asking whether it is held; that
        // lets go within microseconds, an owner does not. An owner also says
        // so in the endpoint file, which a question never writes: once that
        // is there, stop at once, so a process that was going to lose cannot
        // take over the moment the owner it lost to stops.
        let deadline = Instant::now() + PROBE_GRACE;
        let taken = loop {
            match file.try_lock() {
                Ok(()) => break true,
                Err(_) if Instant::now() < deadline && read_endpoint(dir).is_none() => {
                    std::thread::sleep(Duration::from_millis(5));
                }
                Err(_) => break false,
            }
        };
        match taken {
            true => {}
            false => {
                // Someone else holds it. Say who, if they said.
                let existing = read_endpoint(dir);
                return Err(LockError::InUse {
                    pid: existing.as_ref().map_or(0, |e| e.pid),
                    hint: existing
                        .as_ref()
                        .and_then(|e| e.port.map(|p| format!(", serving on 127.0.0.1:{p}"))),
                });
            }
        }

        // The lock was free, so any endpoint file is from an owner that died.
        let _ = std::fs::remove_file(dir.join(ENDPOINT_FILE));

        Ok(DirLock {
            dir: dir.to_path_buf(),
            file,
        })
    }

    /// The directory this lock owns.
    pub fn dir(&self) -> &Path {
        &self.dir
    }

    /// Publish what this process is, and where to reach it.
    ///
    /// Written to a temporary file, flushed, then renamed, so a reader sees
    /// either the old endpoint or the new one and never half of either.
    pub fn publish(&self, endpoint: &Endpoint) -> Result<(), LockError> {
        let final_path = self.dir.join(ENDPOINT_FILE);
        let tmp_path = self.dir.join(format!("{ENDPOINT_FILE}.tmp"));

        let mut json = serde_json::to_string_pretty(endpoint).map_err(|e| LockError::Io {
            context: "cannot render the endpoint file".to_owned(),
            source: std::io::Error::other(e),
        })?;
        json.push('\n');

        {
            let mut file = create_private(&tmp_path)?;
            file.write_all(json.as_bytes())
                .map_err(io(format!("cannot write {}", tmp_path.display())))?;
            file.sync_all()
                .map_err(io(format!("cannot flush {}", tmp_path.display())))?;
        }
        std::fs::rename(&tmp_path, &final_path)
            .map_err(io(format!("cannot install {}", final_path.display())))?;
        sync_dir(&self.dir);
        Ok(())
    }
}

impl Drop for DirLock {
    fn drop(&mut self) {
        // Tidy up so the next process does not have to. If this does not run —
        // because the process was killed — the next `acquire` deletes the
        // endpoint itself, having proved the owner is gone by taking the lock.
        let _ = std::fs::remove_file(self.dir.join(ENDPOINT_FILE));
        let _ = self.file.unlock();
    }
}

/// Create a file only this user can read, as far as the platform allows.
///
/// Unix gets `0600` at creation, so there is no window in which the file is
/// readable. Windows inherits the directory's ACL: inside `%LOCALAPPDATA%`
/// that is already user-scoped, and tightening it further needs `unsafe` Win32
/// calls this phase does not permit. The difference is documented rather than
/// papered over.
fn create_private(path: &Path) -> Result<File, LockError> {
    let mut options = OpenOptions::new();
    options.write(true).create(true).truncate(true);
    #[cfg(unix)]
    {
        use std::os::unix::fs::OpenOptionsExt;
        options.mode(0o600);
    }
    options
        .open(path)
        .map_err(io(format!("cannot create {}", path.display())))
}

/// Flush a directory entry, so a rename survives a power cut.
///
/// A no-op on Windows, which has no handle for a directory that `FlushFileBuffers`
/// accepts; `ReplaceFile`-style atomicity there is provided by the filesystem.
pub fn sync_dir(dir: &Path) {
    #[cfg(unix)]
    {
        if let Ok(handle) = File::open(dir) {
            let _ = handle.sync_all();
        }
    }
    #[cfg(not(unix))]
    {
        let _ = dir;
    }
}

/// Read the endpoint file, if it is there and makes sense.
pub fn read_endpoint(dir: &Path) -> Option<Endpoint> {
    let text = std::fs::read_to_string(dir.join(ENDPOINT_FILE)).ok()?;
    serde_json::from_str(&text).ok()
}

/// Whether a running process owns this data directory.
///
/// Answered by trying the lock: if it can be taken, nobody holds it. The lock
/// is released immediately, so this only reports a moment ago — which is all
/// any such question can report.
pub fn owner(dir: &Path) -> Option<Endpoint> {
    let path = dir.join(LOCK_FILE);
    let file = OpenOptions::new()
        .read(true)
        .write(true)
        .create(false)
        .open(&path)
        .ok()?;
    match file.try_lock() {
        Ok(()) => {
            // It was free: nobody owns it, and any endpoint file is stale.
            // Removed while the lock is still held, so the file of a daemon
            // that takes the lock the moment this lets go is never touched.
            let _ = std::fs::remove_file(dir.join(ENDPOINT_FILE));
            let _ = file.unlock();
            None
        }
        // Held by someone. Only now is the endpoint file worth believing.
        Err(_) => read_endpoint(dir),
    }
}

/// A fresh bearer token: 32 random bytes, hex encoded.
pub fn new_token() -> Result<String, LockError> {
    let mut bytes = [0u8; 32];
    getrandom::fill(&mut bytes).map_err(|e| LockError::Io {
        context: "cannot read random bytes for the daemon token".to_owned(),
        source: std::io::Error::other(e),
    })?;
    Ok(bytes.iter().map(|b| format!("{b:02x}")).collect())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn losing_to_an_owner_that_has_said_so_gives_up_at_once() {
        // The wait is for questions, not owners: a process that loses to a
        // published owner must not linger and take over when that owner stops.
        let dir = tempfile::tempdir().expect("tempdir");
        let owner = DirLock::acquire(dir.path()).expect("first acquire");
        owner
            .publish(&Endpoint::for_this_process())
            .expect("published");
        let started = Instant::now();
        let loser = {
            let path = dir.path().to_path_buf();
            std::thread::spawn(move || DirLock::acquire(&path).map(|_| ()))
        };
        let lost = loser.join().expect("loser");
        assert!(matches!(lost, Err(LockError::InUse { .. })), "{lost:?}");
        assert!(
            started.elapsed() < PROBE_GRACE / 2,
            "the loser waited {:?} for an owner that had said so",
            started.elapsed()
        );
        drop(owner);
    }

    #[test]
    fn asking_who_owns_a_directory_never_stops_a_daemon_taking_it() {
        // One thread asks as fast as it can while another takes the lock,
        // publishes and lets go, over and over: every take must succeed, and
        // a published endpoint must stay published while its owner holds on.
        let dir = tempfile::tempdir().expect("tempdir");
        let path = dir.path().to_path_buf();
        std::fs::write(path.join(LOCK_FILE), b"").expect("lock file");
        let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
        let asker = {
            let path = path.clone();
            let stop = std::sync::Arc::clone(&stop);
            std::thread::spawn(move || {
                let mut asked = 0u64;
                while !stop.load(std::sync::atomic::Ordering::Relaxed) {
                    let _ = owner(&path);
                    asked += 1;
                }
                asked
            })
        };
        for round in 0..200 {
            let held = DirLock::acquire(&path)
                .unwrap_or_else(|e| panic!("round {round}: a question kept the owner out: {e}"));
            let mut endpoint = Endpoint::for_this_process();
            endpoint.port = Some(1);
            held.publish(&endpoint).expect("published");
            for _ in 0..20 {
                assert!(
                    read_endpoint(&path).is_some(),
                    "round {round}: a question removed a held endpoint"
                );
            }
            drop(held);
        }
        stop.store(true, std::sync::atomic::Ordering::Relaxed);
        assert!(asker.join().expect("asker") > 0);
    }

    #[test]
    fn the_lock_keeps_a_second_owner_out_and_names_the_first() {
        let dir = tempfile::tempdir().expect("tempdir");
        let held = DirLock::acquire(dir.path()).expect("first acquire");
        held.publish(&Endpoint::for_this_process())
            .expect("published");

        match DirLock::acquire(dir.path()) {
            Err(LockError::InUse { pid, .. }) => assert_eq!(pid, std::process::id()),
            other => panic!("a second owner got in: {other:?}"),
        }

        drop(held);
        // Once released, the directory is available again.
        DirLock::acquire(dir.path()).expect("acquire after release");
    }

    #[test]
    fn a_released_lock_takes_the_endpoint_file_with_it() {
        let dir = tempfile::tempdir().expect("tempdir");
        let held = DirLock::acquire(dir.path()).expect("acquired");
        held.publish(&Endpoint::for_this_process())
            .expect("published");
        assert!(dir.path().join(ENDPOINT_FILE).exists());

        drop(held);
        assert!(
            !dir.path().join(ENDPOINT_FILE).exists(),
            "a stale endpoint was left behind"
        );
    }

    #[test]
    fn an_endpoint_left_by_a_dead_owner_is_ignored_and_removed() {
        // Simulates the file a killed process leaves: the endpoint is there,
        // but nothing holds the lock. Trusting it would send a client at a
        // port nobody is listening on.
        let dir = tempfile::tempdir().expect("tempdir");
        DirLock::acquire(dir.path())
            .expect("acquired")
            .publish(&Endpoint {
                version: ENDPOINT_VERSION,
                pid: 999_999,
                port: Some(1234),
                token: Some("stale".to_owned()),
                read_token: None,
                started_unix: 0,
                memfork_version: Some(crate::VERSION.to_owned()),
                page_build: None,
            })
            .expect("published");
        // Put it back by hand: dropping the lock above removed it.
        std::fs::write(
            dir.path().join(ENDPOINT_FILE),
            r#"{"version":1,"pid":999999,"port":1234,"token":"stale","started_unix":0}"#,
        )
        .expect("written");

        assert!(
            owner(dir.path()).is_none(),
            "a stale endpoint was reported as a running owner"
        );
        assert!(
            !dir.path().join(ENDPOINT_FILE).exists(),
            "the stale endpoint was not cleaned up"
        );
    }

    #[test]
    fn a_running_owner_is_reported_with_its_details() {
        let dir = tempfile::tempdir().expect("tempdir");
        let held = DirLock::acquire(dir.path()).expect("acquired");
        let mut endpoint = Endpoint::for_this_process();
        endpoint.port = Some(4321);
        endpoint.token = Some("a-token".to_owned());
        held.publish(&endpoint).expect("published");

        let seen = owner(dir.path()).expect("an owner");
        assert_eq!(seen.pid, std::process::id());
        assert_eq!(seen.port, Some(4321));
        assert_eq!(seen.token.as_deref(), Some("a-token"));
    }

    #[test]
    fn the_endpoint_file_is_readable_while_the_lock_is_held() {
        // The reason the lock and the endpoint are separate files: on Windows
        // an exclusive lock on a range blocks reads of it, so a client could
        // not read a token stored inside the locked file.
        let dir = tempfile::tempdir().expect("tempdir");
        let held = DirLock::acquire(dir.path()).expect("acquired");
        held.publish(&Endpoint::for_this_process())
            .expect("published");

        let text = std::fs::read_to_string(dir.path().join(ENDPOINT_FILE))
            .expect("the endpoint file is readable while the lock is held");
        assert!(text.contains("\"pid\""));
    }

    #[test]
    #[cfg(unix)]
    fn the_endpoint_file_is_private_to_this_user() {
        use std::os::unix::fs::PermissionsExt;
        let dir = tempfile::tempdir().expect("tempdir");
        let held = DirLock::acquire(dir.path()).expect("acquired");
        held.publish(&Endpoint::for_this_process())
            .expect("published");

        let mode = std::fs::metadata(dir.path().join(ENDPOINT_FILE))
            .expect("metadata")
            .permissions()
            .mode();
        assert_eq!(mode & 0o777, 0o600, "the token file is readable by others");
    }

    #[test]
    fn the_endpoint_says_which_build_wrote_it() {
        let dir = tempfile::tempdir().expect("tempdir");
        let held = DirLock::acquire(dir.path()).expect("acquired");
        held.publish(&Endpoint::for_this_process())
            .expect("published");
        let seen = read_endpoint(dir.path()).expect("an endpoint");
        assert_eq!(seen.memfork_version.as_deref(), Some(crate::VERSION));
    }

    #[test]
    fn tokens_are_long_and_do_not_repeat() {
        let a = new_token().expect("a token");
        let b = new_token().expect("another token");
        assert_eq!(a.len(), 64);
        assert!(a.chars().all(|c| c.is_ascii_hexdigit()));
        assert_ne!(a, b);
    }

    #[test]
    fn an_unreadable_endpoint_file_is_not_a_crash() {
        let dir = tempfile::tempdir().expect("tempdir");
        std::fs::write(dir.path().join(ENDPOINT_FILE), "not json at all").expect("written");
        assert!(read_endpoint(dir.path()).is_none());
    }
}