second-brain-core 0.13.5

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use std::collections::BTreeMap;
use std::fs;
use std::io::{self, ErrorKind};
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime};

use anyhow::{Context, Result, anyhow, bail};

use crate::private_files::create_private_dir;

const ATTEMPTS: usize = 5;
const RETRY_WAIT: Duration = Duration::from_secs(1);

/// Copies `from` to `to` (which must not exist) only when no file under
/// `from` changed or vanished while it was copied, so the copy is the tree
/// as it stood at one moment. Returns false, leaving nothing at `to`, when
/// the tree kept changing for every attempt.
pub fn copy_dir_stable(from: &Path, to: &Path) -> Result<bool> {
    copy_dir_stable_with(
        from,
        to,
        ATTEMPTS,
        |_| {},
        || std::thread::sleep(RETRY_WAIT),
    )
}

fn copy_dir_stable_with(
    from: &Path,
    to: &Path,
    attempts: usize,
    mut after_copy: impl FnMut(usize),
    mut wait: impl FnMut(),
) -> Result<bool> {
    if fs::symlink_metadata(to).is_ok() {
        bail!("{} already exists", to.display());
    }
    let parent = to
        .parent()
        .ok_or_else(|| anyhow!("{} has no parent directory", to.display()))?;
    let name = to
        .file_name()
        .ok_or_else(|| anyhow!("{} has no name", to.display()))?
        .to_string_lossy();
    for attempt in 0..attempts {
        if attempt > 0 {
            wait();
        }
        let temp = parent.join(format!(".{name}.copying-{}", uuid::Uuid::new_v4().simple()));
        let before = match fingerprint(from) {
            Ok(before) => before,
            Err(e) if e.kind() == ErrorKind::NotFound => continue,
            Err(e) => return Err(e).with_context(|| format!("reading {}", from.display())),
        };
        create_private_dir(&temp)?;
        let copied = copy_tree(from, &temp);
        after_copy(attempt);
        let settled = match copied {
            Ok(()) => fingerprint(from).ok().is_some_and(|after| after == before),
            Err(e) if is_not_found(&e) => false,
            Err(e) => {
                remove_dir(&temp);
                return Err(e);
            }
        };
        if settled {
            sync_dir(&temp)?;
            fs::rename(&temp, to)
                .with_context(|| format!("renaming {} to {}", temp.display(), to.display()))?;
            return Ok(true);
        }
        remove_dir(&temp);
    }
    Ok(false)
}

/// Flushes the copy before returning, so a large backup never holds more
/// than one file's dirty pages in a memory-capped cgroup.
pub fn copy_file_synced(from: &Path, to: &Path) -> Result<()> {
    fs::copy(from, to)
        .with_context(|| format!("copying {} to {}", from.display(), to.display()))?;
    fs::File::open(to)
        .and_then(|f| f.sync_data())
        .with_context(|| format!("syncing {}", to.display()))
}

pub fn sync_dir(dir: &Path) -> Result<()> {
    fs::File::open(dir)
        .and_then(|d| d.sync_all())
        .with_context(|| format!("syncing {}", dir.display()))
}

pub fn is_not_found(e: &anyhow::Error) -> bool {
    e.chain().any(|cause| {
        cause
            .downcast_ref::<io::Error>()
            .is_some_and(|io| io.kind() == ErrorKind::NotFound)
    })
}

#[derive(PartialEq, Eq, Debug)]
enum Entry {
    Dir,
    File { len: u64, modified: SystemTime },
    Link(PathBuf),
}

fn fingerprint(root: &Path) -> io::Result<BTreeMap<PathBuf, Entry>> {
    let mut seen = BTreeMap::new();
    let mut pending = vec![PathBuf::new()];
    while let Some(rel) = pending.pop() {
        for entry in fs::read_dir(root.join(&rel))? {
            let entry = entry?;
            let path = rel.join(entry.file_name());
            let kind = entry.file_type()?;
            let seen_as = if kind.is_dir() {
                pending.push(path.clone());
                Entry::Dir
            } else if kind.is_symlink() {
                Entry::Link(fs::read_link(entry.path())?)
            } else {
                let meta = entry.metadata()?;
                Entry::File {
                    len: meta.len(),
                    modified: meta.modified()?,
                }
            };
            seen.insert(path, seen_as);
        }
    }
    Ok(seen)
}

fn copy_tree(from: &Path, to: &Path) -> Result<()> {
    for entry in fs::read_dir(from).with_context(|| format!("reading {}", from.display()))? {
        let entry = entry?;
        let source = entry.path();
        let target = to.join(entry.file_name());
        let kind = entry.file_type()?;
        if kind.is_dir() {
            create_private_dir(&target)?;
            copy_tree(&source, &target)?;
            sync_dir(&target)?;
        } else if kind.is_symlink() {
            copy_symlink(&source, &target)?;
        } else if kind.is_file() {
            copy_file_synced(&source, &target)?;
        }
    }
    Ok(())
}

#[cfg(unix)]
pub fn copy_symlink(source: &Path, target: &Path) -> Result<()> {
    let link = fs::read_link(source).with_context(|| format!("reading {}", source.display()))?;
    std::os::unix::fs::symlink(&link, target)
        .with_context(|| format!("linking {} to {}", target.display(), link.display()))
}

#[cfg(not(unix))]
pub fn copy_symlink(source: &Path, _target: &Path) -> Result<()> {
    tracing::warn!("the copy skips the link {}", source.display());
    Ok(())
}

fn remove_dir(dir: &Path) {
    if let Err(e) = fs::remove_dir_all(dir)
        && e.kind() != ErrorKind::NotFound
    {
        tracing::warn!("cannot remove the partial copy {}: {e}", dir.display());
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn tree(root: &Path) -> PathBuf {
        let src = root.join("src");
        fs::create_dir_all(src.join("nested")).unwrap();
        fs::write(src.join("meta.json"), b"one").unwrap();
        fs::write(src.join("nested/segment"), b"segment bytes").unwrap();
        src
    }

    fn read(path: &Path) -> Vec<u8> {
        fs::read(path).unwrap()
    }

    fn leftovers(root: &Path) -> Vec<String> {
        fs::read_dir(root)
            .unwrap()
            .map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
            .filter(|n| n.contains(".copying-"))
            .collect()
    }

    #[test]
    fn a_quiet_tree_is_copied_whole() {
        let root = tempfile::tempdir().unwrap();
        let src = tree(root.path());
        let dest = root.path().join("dest");
        assert!(copy_dir_stable(&src, &dest).unwrap());
        assert_eq!(read(&dest.join("meta.json")), b"one");
        assert_eq!(read(&dest.join("nested/segment")), b"segment bytes");
        assert!(leftovers(root.path()).is_empty());
    }

    #[test]
    fn a_tree_changed_during_the_copy_is_copied_again() {
        let root = tempfile::tempdir().unwrap();
        let src = tree(root.path());
        let dest = root.path().join("dest");
        let mut waits = 0;
        let copied = copy_dir_stable_with(
            &src,
            &dest,
            5,
            |attempt| {
                if attempt == 0 {
                    fs::write(src.join("meta.json"), b"two, longer").unwrap();
                }
            },
            || waits += 1,
        )
        .unwrap();
        assert!(copied);
        assert_eq!(waits, 1);
        assert_eq!(read(&dest.join("meta.json")), b"two, longer");
        assert!(leftovers(root.path()).is_empty());
    }

    #[test]
    fn a_file_deleted_during_the_copy_is_a_change_not_an_error() {
        let root = tempfile::tempdir().unwrap();
        let src = tree(root.path());
        let dest = root.path().join("dest");
        let copied = copy_dir_stable_with(
            &src,
            &dest,
            5,
            |attempt| {
                if attempt == 0 {
                    fs::remove_file(src.join("nested/segment")).unwrap();
                }
            },
            || {},
        )
        .unwrap();
        assert!(copied);
        assert!(!dest.join("nested/segment").exists());
    }

    #[test]
    fn a_tree_missing_when_an_attempt_starts_is_tried_again() {
        let root = tempfile::tempdir().unwrap();
        let src = tree(root.path());
        let dest = root.path().join("dest");
        let mut waits = 0;
        let copied = copy_dir_stable_with(
            &src,
            &dest,
            5,
            |attempt| {
                if attempt == 0 {
                    fs::remove_dir_all(&src).unwrap();
                }
            },
            || {
                waits += 1;
                if waits == 2 {
                    tree(root.path());
                }
            },
        )
        .unwrap();
        assert!(copied);
        assert_eq!(waits, 2);
        assert_eq!(read(&dest.join("meta.json")), b"one");
    }

    #[test]
    fn a_tree_that_never_settles_leaves_nothing_behind() {
        let root = tempfile::tempdir().unwrap();
        let src = tree(root.path());
        let dest = root.path().join("dest");
        let mut n = 0;
        let copied = copy_dir_stable_with(
            &src,
            &dest,
            3,
            |_| {
                n += 1;
                fs::write(src.join("meta.json"), "x".repeat(n)).unwrap();
            },
            || {},
        )
        .unwrap();
        assert!(!copied);
        assert_eq!(n, 3);
        assert!(!dest.exists());
        assert!(leftovers(root.path()).is_empty());
    }

    #[test]
    fn an_existing_destination_is_refused() {
        let root = tempfile::tempdir().unwrap();
        let src = tree(root.path());
        let dest = root.path().join("dest");
        fs::create_dir(&dest).unwrap();
        assert!(copy_dir_stable(&src, &dest).is_err());
    }
}