use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::sync::mpsc::{self, Receiver, RecvTimeoutError, Sender};
use std::sync::{Arc, Mutex, OnceLock};
use std::time::{Duration, Instant, SystemTime};
use std::{fs, io};
use blit_remote::fs::{
FS_CLOSED_CLIENT_REQUEST, FS_CLOSED_RESOURCE_LIMIT, FS_CLOSED_ROOT_GONE, FS_DONE_CONFLICT,
FS_DONE_INVALID, FS_DONE_NOT_FOUND, FS_DONE_OK, FS_DONE_OTHER, FS_DONE_PERMISSION,
FS_DONE_TOO_LARGE, FS_DONE_WRONG_TYPE, FS_ENTRY_DIR, FS_ENTRY_FILE, FS_ENTRY_FILTERED,
FS_ENTRY_LINK_DIR, FS_ENTRY_NO_CONTENT, FS_ENTRY_OTHER, FS_ENTRY_SYMLINK, FS_ENTRY_TYPE_MASK,
FS_ENTRY_UNREADABLE, FS_ENTRY_UNSTABLE, FS_FILE_NOT_FOUND, FS_FILE_OK, FS_FILE_UNREADABLE,
FS_OP_HARDLINK, FS_OP_MKDIR, FS_OP_MKPARENTS, FS_OP_NO_CAS, FS_OP_REMOVE, FS_OP_RENAME,
FS_OP_SYMLINK, FS_UPDATE_RESET, FS_UPDATE_SYNC, FS_WRITE_DURABLE, FS_WRITE_FOLLOW_SYMLINK,
FS_WRITE_MKPARENTS, FS_WRITE_NO_CAS, FsContent, FsRecord, append_fs_record, msg_fs_closed,
msg_fs_done, msg_fs_file, msg_fs_update,
};
pub mod backend;
pub mod ignores;
pub use ignores::{IgnoreSpec, MAX_PATTERNS as MAX_IGNORE_PATTERNS};
#[derive(Clone, Debug)]
pub struct SyncOptions {
pub recursive: bool,
pub content: bool,
pub cross_filesystem: bool,
pub latency: Duration,
pub inline_max: u64,
pub window_bytes: usize,
pub batch_target: usize,
pub max_entries: usize,
}
impl Default for SyncOptions {
fn default() -> Self {
Self {
recursive: true,
content: false,
cross_filesystem: false,
latency: env_ms("BLIT_FS_LATENCY_MS", 20),
inline_max: env_u64("BLIT_FS_INLINE_MAX", 16 * 1024 * 1024),
window_bytes: env_u64("BLIT_FS_WINDOW", 1024 * 1024) as usize,
batch_target: 64 * 1024,
max_entries: env_u64("BLIT_FS_MAX_ENTRIES", 1_000_000) as usize,
}
}
}
fn env_ms(name: &str, default: u64) -> Duration {
Duration::from_millis(env_u64(name, default).clamp(1, 1000))
}
fn env_u64(name: &str, default: u64) -> u64 {
std::env::var(name)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
#[derive(Clone, Debug)]
pub enum Hint {
Dirty(PathBuf),
Rescan,
}
#[derive(Debug)]
pub struct InflightGuard {
set: Arc<Mutex<std::collections::HashSet<u16>>>,
nonce: u16,
}
impl InflightGuard {
pub fn new(set: Arc<Mutex<std::collections::HashSet<u16>>>, nonce: u16) -> Self {
InflightGuard { set, nonce }
}
}
impl Drop for InflightGuard {
fn drop(&mut self) {
if let Ok(mut set) = self.set.lock() {
set.remove(&self.nonce);
}
}
}
#[derive(Clone, Debug)]
pub struct WriteReq {
pub nonce: u16,
pub path: String,
pub base: u128,
pub mode: u32,
pub flags: u8,
pub content_kind: u8,
pub content: Vec<u8>,
pub inflight: Option<Arc<InflightGuard>>,
}
#[derive(Clone, Debug)]
pub struct OpReq {
pub nonce: u16,
pub op: u8,
pub a: String,
pub b: String,
pub base: u128,
pub mode: u32,
pub flags: u8,
pub inflight: Option<Arc<InflightGuard>>,
}
#[derive(Clone, Debug)]
pub enum Command {
Ack(u32),
Fetch { nonce: u16, path: String },
Write(WriteReq),
Op(OpReq),
Stop,
}
pub trait BackendHandle: Send {
fn add_dir(&self, _dir: &Path) -> bool {
true
}
fn watch_outside(&self, _dir: &Path) {}
fn remove_dir(&self, _dir: &Path) {}
fn retain_dirs(&self, _keep: &dyn Fn(&Path) -> bool) {}
}
pub struct NoopBackend;
impl BackendHandle for NoopBackend {}
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub struct RootKey {
pub path: PathBuf,
pub recursive: bool,
pub cross_filesystem: bool,
pub ignores: IgnoreSpec,
}
enum RootMsg {
Hint(Hint),
Subscribe {
id: u64,
tx: Sender<SyncMsg>,
latency: Duration,
},
Unsubscribe {
id: u64,
},
HashLearned {
path: String,
meta: NodeMeta,
},
}
enum RootUpdate {
Snapshot {
index: Arc<Index>,
settled: Option<Instant>,
changed: Option<Arc<std::collections::BTreeSet<String>>>,
recheck: Arc<std::collections::BTreeSet<String>>,
},
Closed(u8),
}
enum SyncMsg {
Cmd(Command),
Root(RootUpdate),
}
pub struct SharedRootHandle {
key: RootKey,
single: bool,
tx: Sender<RootMsg>,
closed: Arc<OnceLock<u8>>,
learned: Mutex<std::collections::HashMap<String, NodeMeta>>,
_backend: Mutex<Option<backend::WatchBackend>>,
}
impl SharedRootHandle {
pub fn key(&self) -> &RootKey {
&self.key
}
pub fn is_single(&self) -> bool {
self.single
}
pub fn hint_sender(&self) -> HintSender {
HintSender {
tx: self.tx.clone(),
}
}
fn is_closed(&self) -> bool {
self.closed.get().is_some()
}
}
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
struct RegKey {
root: RootKey,
single: bool,
}
type Registry = std::collections::HashMap<RegKey, std::sync::Weak<SharedRootHandle>>;
fn registry() -> &'static Mutex<Registry> {
static REGISTRY: OnceLock<Mutex<Registry>> = OnceLock::new();
REGISTRY.get_or_init(Default::default)
}
pub fn open_root(key: RootKey) -> Result<Arc<SharedRootHandle>, (u8, String)> {
open_root_inner(key, false, true)
}
pub fn open_root_unwatched(key: RootKey) -> Arc<SharedRootHandle> {
open_root_inner(key, false, false).expect("unwatched open cannot fail")
}
pub fn open_single_root(path: PathBuf) -> Result<Arc<SharedRootHandle>, (u8, String)> {
open_root_inner(single_root_key(path), true, true)
}
pub fn open_single_root_unwatched(path: PathBuf) -> Arc<SharedRootHandle> {
open_root_inner(single_root_key(path), true, false).expect("unwatched open cannot fail")
}
fn single_root_key(path: PathBuf) -> RootKey {
RootKey {
path,
recursive: false,
cross_filesystem: false,
ignores: IgnoreSpec::default(),
}
}
fn watch_error_status(err: ¬ify::Error) -> u8 {
use blit_remote::fs::{
FS_STATUS_NOT_FOUND, FS_STATUS_OTHER, FS_STATUS_PERMISSION_DENIED, FS_STATUS_RESOURCE_LIMIT,
};
match &err.kind {
notify::ErrorKind::MaxFilesWatch => FS_STATUS_RESOURCE_LIMIT,
notify::ErrorKind::PathNotFound => FS_STATUS_NOT_FOUND,
notify::ErrorKind::Io(e) => match e.raw_os_error() {
Some(23) | Some(24) | Some(28) => FS_STATUS_RESOURCE_LIMIT,
_ => match e.kind() {
io::ErrorKind::PermissionDenied => FS_STATUS_PERMISSION_DENIED,
io::ErrorKind::NotFound => FS_STATUS_NOT_FOUND,
_ => FS_STATUS_OTHER,
},
},
_ => FS_STATUS_OTHER,
}
}
fn open_root_inner(
key: RootKey,
single: bool,
watched: bool,
) -> Result<Arc<SharedRootHandle>, (u8, String)> {
let reg_key = RegKey {
root: key.clone(),
single,
};
{
let mut map = registry().lock().unwrap();
map.retain(|_, weak| weak.strong_count() > 0);
if let Some(existing) = map
.get(®_key)
.and_then(std::sync::Weak::upgrade)
.filter(|h| !h.is_closed())
{
return Ok(existing);
}
}
let (tx, rx) = mpsc::channel();
let backend = if watched {
let hints = HintSender { tx: tx.clone() };
let (watch_path, recursive) = if single {
let parent = key
.path
.parent()
.ok_or_else(|| {
use blit_remote::fs::FS_STATUS_OTHER;
(FS_STATUS_OTHER, "single root has no parent".to_string())
})?
.to_path_buf();
(parent, false)
} else {
(key.path.clone(), key.recursive)
};
let per_dir =
backend::per_dir_watching_pays(key.recursive, single, !key.ignores.is_empty());
Some(
backend::watch(&watch_path, recursive, per_dir, hints)
.map_err(|e| (watch_error_status(&e), e.to_string()))?,
)
} else {
None
};
let registrar: Box<dyn BackendHandle> = match &backend {
Some(backend) => Box::new(backend.watches.clone()),
None => Box::new(NoopBackend),
};
let mut map = registry().lock().unwrap();
map.retain(|_, weak| weak.strong_count() > 0);
if let Some(existing) = map
.get(®_key)
.and_then(std::sync::Weak::upgrade)
.filter(|h| !h.is_closed())
{
return Ok(existing);
}
let closed: Arc<OnceLock<u8>> = Arc::new(OnceLock::new());
let handle = Arc::new(SharedRootHandle {
key: key.clone(),
single,
tx,
closed: closed.clone(),
learned: Mutex::new(Default::default()),
_backend: Mutex::new(backend),
});
std::thread::Builder::new()
.name("blit-fsroot".into())
.spawn(move || Reconciler::new(key, single, rx, registrar, closed).run())
.expect("spawn fssync reconciler");
map.insert(reg_key, Arc::downgrade(&handle));
Ok(handle)
}
pub struct SyncHandle {
tx: Sender<SyncMsg>,
done: Arc<std::sync::atomic::AtomicBool>,
}
impl SyncHandle {
pub fn command(&self, cmd: Command) -> bool {
self.tx.send(SyncMsg::Cmd(cmd)).is_ok()
}
pub fn is_done(&self) -> bool {
self.done.load(std::sync::atomic::Ordering::Acquire)
}
}
impl Drop for SyncHandle {
fn drop(&mut self) {
let _ = self.tx.send(SyncMsg::Cmd(Command::Stop));
}
}
#[derive(Clone)]
pub struct HintSender {
tx: Sender<RootMsg>,
}
impl HintSender {
pub fn send(&self, hint: Hint) -> bool {
self.tx.send(RootMsg::Hint(hint)).is_ok()
}
}
pub type Outbox = Box<dyn FnMut(Vec<u8>) -> bool + Send>;
pub fn validate_root(path: &str) -> Result<PathBuf, (u8, String)> {
use blit_remote::fs::{FS_STATUS_NOT_FOUND, FS_STATUS_OTHER, FS_STATUS_PERMISSION_DENIED};
if path.is_empty() || path.contains('\0') {
return Err((FS_STATUS_OTHER, "invalid path".into()));
}
let err = match fs::canonicalize(path) {
Ok(p) => return Ok(p),
Err(e) => e,
};
if err.kind() == io::ErrorKind::NotFound
&& path.contains('%')
&& let Some(decoded) = wire_to_os(path)
&& let Ok(p) = fs::canonicalize(&decoded)
{
return Ok(p);
}
let status = match err.kind() {
io::ErrorKind::NotFound => FS_STATUS_NOT_FOUND,
io::ErrorKind::PermissionDenied => FS_STATUS_PERMISSION_DENIED,
_ => FS_STATUS_OTHER,
};
Err((status, err.to_string()))
}
pub fn validate_single_root(path: &str) -> Result<PathBuf, (u8, String)> {
use blit_remote::fs::FS_STATUS_OTHER;
let canon = validate_root(path)?;
match fs::symlink_metadata(&canon) {
Ok(md) if md.is_dir() => Err((
FS_STATUS_OTHER,
"single sync root is a directory".to_string(),
)),
_ => Ok(canon),
}
}
pub fn start_sync(
shared: &Arc<SharedRootHandle>,
sync_id: u16,
opts: SyncOptions,
outbox: Outbox,
) -> SyncHandle {
static SUB_IDS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
let sub_id = SUB_IDS.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let (tx, rx) = mpsc::channel();
let _ = shared.tx.send(RootMsg::Subscribe {
id: sub_id,
tx: tx.clone(),
latency: opts.latency,
});
let engine = SyncEngine::new(sync_id, shared.clone(), sub_id, opts, rx, outbox);
let done = Arc::new(std::sync::atomic::AtomicBool::new(false));
let done_thread = done.clone();
std::thread::Builder::new()
.name(format!("blit-fssync-{sync_id}"))
.spawn(move || {
engine.run();
done_thread.store(true, std::sync::atomic::Ordering::Release);
})
.expect("spawn fssync engine");
SyncHandle { tx, done }
}
pub fn escape_bytes(bytes: &[u8]) -> String {
let mut out = String::with_capacity(bytes.len());
let mut rest = bytes;
loop {
match std::str::from_utf8(rest) {
Ok(s) => {
push_escaping_percent(&mut out, s);
return out;
}
Err(e) => {
let (valid, after) = rest.split_at(e.valid_up_to());
push_escaping_percent(&mut out, unsafe { std::str::from_utf8_unchecked(valid) });
let bad = e.error_len().unwrap_or(after.len());
for &b in &after[..bad] {
out.push_str(&format!("%{b:02X}"));
}
rest = &after[bad..];
}
}
}
}
fn push_escaping_percent(out: &mut String, s: &str) {
for ch in s.chars() {
if ch == '%' {
out.push_str("%25");
} else {
out.push(ch);
}
}
}
pub fn unescape_to_bytes(s: &str) -> Option<Vec<u8>> {
let mut out = Vec::with_capacity(s.len());
let bytes = s.as_bytes();
let mut i = 0;
while i < bytes.len() {
if bytes[i] == b'%' {
let hex = bytes.get(i + 1..i + 3)?;
let hi = (hex[0] as char).to_digit(16)?;
let lo = (hex[1] as char).to_digit(16)?;
out.push((hi * 16 + lo) as u8);
i += 3;
} else {
out.push(bytes[i]);
i += 1;
}
}
Some(out)
}
pub fn escape_wide(units: &[u16]) -> String {
let mut out = String::with_capacity(units.len());
for decoded in char::decode_utf16(units.iter().copied()) {
match decoded {
Ok('%') => out.push_str("%25"),
Ok(c) => out.push(c),
Err(e) => {
out.push_str(&format!("%u{:04X}", e.unpaired_surrogate()));
}
}
}
out
}
pub fn unescape_to_wide(s: &str) -> Option<Vec<u16>> {
let mut out = Vec::with_capacity(s.len());
let bytes = s.as_bytes();
let mut i = 0;
while i < bytes.len() {
if bytes[i] == b'%' {
if bytes.get(i + 1) == Some(&b'u') {
out.push(u16::from_str_radix(s.get(i + 2..i + 6)?, 16).ok()?);
i += 6;
} else {
out.push(u16::from(
u8::from_str_radix(s.get(i + 1..i + 3)?, 16).ok()?,
));
i += 3;
}
} else {
let c = s[i..].chars().next()?;
let mut buf = [0u16; 2];
out.extend_from_slice(c.encode_utf16(&mut buf));
i += c.len_utf8();
}
}
Some(out)
}
#[cfg(unix)]
pub fn escape_path(path: &Path) -> String {
use std::os::unix::ffi::OsStrExt;
escape_bytes(path.as_os_str().as_bytes())
}
#[cfg(windows)]
pub fn escape_path(path: &Path) -> String {
use std::os::windows::ffi::OsStrExt;
escape_wide(&path.as_os_str().encode_wide().collect::<Vec<_>>())
}
#[cfg(all(not(unix), not(windows)))]
pub fn escape_path(path: &Path) -> String {
escape_bytes(path.to_string_lossy().as_bytes())
}
#[cfg(unix)]
fn os_to_wire(name: &std::ffi::OsStr) -> String {
use std::os::unix::ffi::OsStrExt;
escape_bytes(name.as_bytes())
}
#[cfg(windows)]
fn os_to_wire(name: &std::ffi::OsStr) -> String {
use std::os::windows::ffi::OsStrExt;
escape_wide(&name.encode_wide().collect::<Vec<_>>())
}
#[cfg(all(not(unix), not(windows)))]
fn os_to_wire(name: &std::ffi::OsStr) -> String {
escape_bytes(name.to_string_lossy().as_bytes())
}
#[cfg(unix)]
fn wire_to_os(component: &str) -> Option<std::ffi::OsString> {
use std::os::unix::ffi::OsStringExt;
Some(std::ffi::OsString::from_vec(unescape_to_bytes(component)?))
}
#[cfg(windows)]
fn wire_to_os(component: &str) -> Option<std::ffi::OsString> {
use std::os::windows::ffi::OsStringExt;
Some(std::ffi::OsString::from_wide(&unescape_to_wide(component)?))
}
#[cfg(all(not(unix), not(windows)))]
fn wire_to_os(component: &str) -> Option<std::ffi::OsString> {
Some(
String::from_utf8(unescape_to_bytes(component)?)
.ok()?
.into(),
)
}
pub fn resolve_wire_path(root: &Path, wire: &str) -> Option<PathBuf> {
use std::path::Component;
let mut abs = root.to_path_buf();
if wire.is_empty() {
return Some(abs);
}
for component in wire.split('/') {
let os = wire_to_os(component)?;
let mut parts = Path::new(&os).components();
match (parts.next(), parts.next()) {
(Some(Component::Normal(part)), None) if part == os.as_os_str() => abs.push(part),
_ => return None,
}
}
Some(abs)
}
fn join_wire(parent: &str, child: &str) -> String {
if parent.is_empty() {
child.to_string()
} else {
format!("{parent}/{child}")
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct NodeMeta {
pub node_type: u8,
pub size: u64,
pub mtime_ns: u64,
pub mode: u32,
pub hash: u128,
pub dev_ino: (u64, u64),
pub link_dir: bool,
pub filtered: bool,
}
impl NodeMeta {
fn enumerable_dir(&self) -> bool {
self.node_type == FS_ENTRY_DIR || (self.node_type == FS_ENTRY_SYMLINK && self.link_dir)
}
fn same_identity(&self, other: &NodeMeta) -> bool {
self.node_type == other.node_type && self.dev_ino != (0, 0) && self.dev_ino == other.dev_ino
}
fn content_changed(&self, prev: &NodeMeta) -> bool {
self.node_type != prev.node_type
|| self.size != prev.size
|| self.mtime_ns != prev.mtime_ns
|| self.dev_ino != prev.dev_ino
}
fn visible_eq(&self, other: &NodeMeta) -> bool {
self.node_type == other.node_type
&& self.size == other.size
&& self.mtime_ns == other.mtime_ns
&& self.mode == other.mode
&& self.dev_ino == other.dev_ino
&& self.filtered == other.filtered
&& self.link_dir == other.link_dir
}
}
fn target_identity(md: &fs::Metadata) -> (u64, u64) {
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
(md.dev(), md.ino())
}
#[cfg(not(unix))]
{
let _ = md;
(0, 0)
}
}
fn stat_meta(path: &Path) -> io::Result<NodeMeta> {
let md = fs::symlink_metadata(path)?;
let ft = md.file_type();
let node_type = if ft.is_file() {
FS_ENTRY_FILE
} else if ft.is_dir() {
FS_ENTRY_DIR
} else if ft.is_symlink() {
FS_ENTRY_SYMLINK
} else {
FS_ENTRY_OTHER
};
let mtime_ns = md
.modified()
.ok()
.and_then(|t| t.duration_since(SystemTime::UNIX_EPOCH).ok())
.map(|d| d.as_nanos() as u64)
.unwrap_or(0);
#[cfg(unix)]
let (mode, dev_ino) = {
use std::os::unix::fs::MetadataExt;
(md.mode(), (md.dev(), md.ino()))
};
#[cfg(not(unix))]
let (mode, dev_ino) = (0u32, (0u64, 0u64));
Ok(NodeMeta {
node_type,
link_dir: ft.is_symlink() && fs::metadata(path).map(|m| m.is_dir()).unwrap_or(false),
filtered: false,
size: if ft.is_file() || ft.is_symlink() {
md.len()
} else {
0
},
mtime_ns,
mode,
hash: 0,
dev_ino,
})
}
type Index = BTreeMap<String, NodeMeta>;
fn is_under(path: &str, root: &str) -> bool {
root.is_empty()
|| path == root
|| (path.len() > root.len()
&& path.starts_with(root)
&& path.as_bytes()[root.len()] == b'/')
}
fn subtree_keys<V>(map: &BTreeMap<String, V>, root: &str) -> Vec<String> {
if root.is_empty() {
return map.keys().cloned().collect();
}
let mut keys: Vec<String> = Vec::new();
if map.contains_key(root) {
keys.push(root.to_string());
}
let prefix = format!("{root}/");
keys.extend(
map.range(prefix.clone()..)
.take_while(|(k, _)| k.starts_with(&prefix))
.map(|(k, _)| k.clone()),
);
keys
}
fn subtree_entries<'a, V>(
map: &'a BTreeMap<String, V>,
root: &str,
) -> impl Iterator<Item = (&'a String, &'a V)> {
let own = if root.is_empty() {
None
} else {
map.get_key_value(root)
};
let prefix = if root.is_empty() {
String::new()
} else {
format!("{root}/")
};
own.into_iter().chain(
map.range(prefix.clone()..)
.take_while(move |(k, _)| k.starts_with(&prefix)),
)
}
fn parent_wire(rel: &str) -> Option<&str> {
if rel.is_empty() {
None
} else {
Some(match rel.rfind('/') {
Some(i) => &rel[..i],
None => "",
})
}
}
fn rebase_subtree_path(path: &str, from: &str, to: &str) -> String {
let suffix = if path.len() > from.len() {
&path[from.len() + usize::from(!from.is_empty())..]
} else {
""
};
if suffix.is_empty() {
to.to_string()
} else if to.is_empty() {
suffix.to_string()
} else {
format!("{to}/{suffix}")
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum DiffOp {
Upsert {
path: String,
content_changed: bool,
},
Delete {
path: String,
},
Move {
from: String,
to: String,
},
}
pub fn diff(prev: &Index, curr: &Index) -> Vec<DiffOp> {
let mut removed: Vec<&String> = Vec::new();
let mut added: Vec<&String> = Vec::new();
let mut changed: Vec<(&String, bool)> = Vec::new();
let mut pi = prev.iter().peekable();
let mut ci = curr.iter().peekable();
loop {
match (pi.peek(), ci.peek()) {
(Some((pk, pv)), Some((ck, cv))) => {
if pk == ck {
if !cv.visible_eq(pv) {
changed.push((ck, cv.content_changed(pv)));
}
pi.next();
ci.next();
} else if pk < ck {
removed.push(pk);
pi.next();
} else {
added.push(ck);
ci.next();
}
}
(Some((pk, _)), None) => {
removed.push(pk);
pi.next();
}
(None, Some((ck, _))) => {
added.push(ck);
ci.next();
}
(None, None) => break,
}
}
diff_classified(prev, curr, removed, added, changed)
}
fn diff_changed(
prev: &Index,
curr: &Index,
changed_keys: &std::collections::BTreeSet<String>,
) -> Vec<DiffOp> {
let mut removed: Vec<&String> = Vec::new();
let mut added: Vec<&String> = Vec::new();
let mut changed: Vec<(&String, bool)> = Vec::new();
for key in changed_keys {
match (prev.get_key_value(key), curr.get_key_value(key)) {
(Some((pk, pv)), Some((_, cv))) => {
if !cv.visible_eq(pv) {
changed.push((pk, cv.content_changed(pv)));
}
}
(Some((pk, _)), None) => removed.push(pk),
(None, Some((ck, _))) => added.push(ck),
(None, None) => {}
}
}
diff_classified(prev, curr, removed, added, changed)
}
fn cover_sorted(paths: &[&String], covered: &mut [bool], root: &str) {
if let Ok(i) = paths.binary_search_by(|p| p.as_str().cmp(root)) {
covered[i] = true;
}
let prefix = format!("{root}/");
let start = paths.partition_point(|p| p.as_str() < prefix.as_str());
for i in start..paths.len() {
if !paths[i].starts_with(&prefix) {
break;
}
covered[i] = true;
}
}
fn diff_classified(
prev: &Index,
curr: &Index,
removed: Vec<&String>,
added: Vec<&String>,
changed: Vec<(&String, bool)>,
) -> Vec<DiffOp> {
let mut moves: Vec<(String, String)> = Vec::new();
let mut removed_covered = vec![false; removed.len()];
let mut added_covered = vec![false; added.len()];
let mut by_identity: std::collections::HashMap<(u64, u64), usize> =
std::collections::HashMap::new();
for (idx, path) in removed.iter().enumerate() {
let meta = &prev[*path];
if meta.dev_ino != (0, 0) {
by_identity.insert(meta.dev_ino, idx);
}
}
let mut add_order: Vec<usize> = (0..added.len()).collect();
add_order.sort_by_key(|&i| added[i].len());
for ai in add_order {
if added_covered[ai] {
continue;
}
let to = added[ai];
let cmeta = &curr[to];
let Some(&ri) = by_identity.get(&cmeta.dev_ino) else {
continue;
};
if removed_covered[ri] || !prev[removed[ri]].same_identity(cmeta) {
continue;
}
let from = removed[ri];
cover_sorted(&removed, &mut removed_covered, from);
cover_sorted(&added, &mut added_covered, to);
moves.push((from.clone(), to.clone()));
}
let mut ops = Vec::new();
for (from, to) in &moves {
ops.push(DiffOp::Move {
from: from.clone(),
to: to.clone(),
});
}
let mut emitted: std::collections::HashSet<&str> = std::collections::HashSet::new();
for (i, path) in removed.iter().enumerate() {
if removed_covered[i] {
continue;
}
let mut ancestor_deleted = false;
let mut cursor: &str = path;
while let Some(parent) = parent_wire(cursor) {
if emitted.contains(parent) {
ancestor_deleted = true;
break;
}
cursor = parent;
}
if !ancestor_deleted {
emitted.insert(path.as_str());
ops.push(DiffOp::Delete {
path: (*path).clone(),
});
}
}
for (i, path) in added.iter().enumerate() {
if !added_covered[i] {
ops.push(DiffOp::Upsert {
path: (*path).clone(),
content_changed: true,
});
}
}
for (from, to) in &moves {
for (path, _) in subtree_entries(prev, from) {
let new_path = rebase_subtree_path(path, from, to);
if !curr.contains_key(&new_path) {
ops.push(DiffOp::Delete { path: new_path });
}
}
for (path, new) in subtree_entries(curr, to) {
let old_path = rebase_subtree_path(path, to, from);
match prev.get(&old_path) {
Some(old) if new.visible_eq(old) => {}
Some(old) => ops.push(DiffOp::Upsert {
path: path.clone(),
content_changed: new.content_changed(old),
}),
None => ops.push(DiffOp::Upsert {
path: path.clone(),
content_changed: true,
}),
}
}
}
for (path, content_changed) in changed {
ops.push(DiffOp::Upsert {
path: path.clone(),
content_changed,
});
}
ops
}
pub enum ReadOutcome {
Stable(Vec<u8>),
Unstable,
Unreadable,
}
enum ReadMetaOutcome {
Stable(Vec<u8>, NodeMeta),
Unstable,
Unreadable,
}
fn read_verified_meta(path: &Path) -> ReadMetaOutcome {
for _ in 0..2 {
let Ok(before) = stat_meta(path) else {
return ReadMetaOutcome::Unreadable;
};
let read = if before.node_type == FS_ENTRY_SYMLINK {
link_target_bytes(path)
} else {
fs::read(path)
};
let Ok(data) = read else {
return ReadMetaOutcome::Unreadable;
};
match stat_meta(path) {
Ok(after)
if after.dev_ino == before.dev_ino
&& after.size == before.size
&& after.mtime_ns == before.mtime_ns =>
{
return ReadMetaOutcome::Stable(data, after);
}
Ok(_) => continue,
Err(_) => return ReadMetaOutcome::Unreadable,
}
}
ReadMetaOutcome::Unstable
}
pub fn read_verified(path: &Path) -> ReadOutcome {
match read_verified_meta(path) {
ReadMetaOutcome::Stable(data, _) => ReadOutcome::Stable(data),
ReadMetaOutcome::Unstable => ReadOutcome::Unstable,
ReadMetaOutcome::Unreadable => ReadOutcome::Unreadable,
}
}
const RACY_WINDOW_NS: u64 = 2_000_000_000;
fn racily_clean(mtime_ns: u64) -> bool {
let now_ns = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.map(|d| d.as_nanos() as u64)
.unwrap_or(0);
now_ns.saturating_sub(mtime_ns) < RACY_WINDOW_NS
}
pub fn blake3_128(data: &[u8]) -> u128 {
let hash = blake3::hash(data);
u128::from_le_bytes(hash.as_bytes()[..16].try_into().unwrap())
}
fn fs_write_max() -> u64 {
std::env::var("BLIT_FS_WRITE_MAX")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(16 * 1024 * 1024)
}
fn write_io_status(e: &io::Error) -> u8 {
match e.kind() {
io::ErrorKind::NotFound => FS_DONE_NOT_FOUND,
io::ErrorKind::PermissionDenied => FS_DONE_PERMISSION,
io::ErrorKind::AlreadyExists => FS_DONE_CONFLICT,
_ => FS_DONE_OTHER,
}
}
enum SymlinkPolicy {
Refuse,
Follow,
Operate,
}
enum ConfineError {
Invalid,
Io(io::Error),
Escapes,
}
fn confine_target(root: &Path, wire: &str) -> Result<PathBuf, ConfineError> {
let abs = resolve_wire_path(root, wire).ok_or(ConfineError::Invalid)?;
let (Some(parent), Some(name)) = (abs.parent(), abs.file_name()) else {
return Err(ConfineError::Invalid);
};
let canon_parent = fs::canonicalize(parent).map_err(ConfineError::Io)?;
if !canon_parent.starts_with(root) {
return Err(ConfineError::Escapes);
}
Ok(canon_parent.join(name))
}
fn resolve_write_target(root: &Path, wire: &str, policy: SymlinkPolicy) -> Result<PathBuf, u8> {
let target = match confine_target(root, wire) {
Ok(t) => t,
Err(ConfineError::Invalid) => return Err(FS_DONE_INVALID),
Err(ConfineError::Io(e)) => return Err(write_io_status(&e)),
Err(ConfineError::Escapes) => return Err(FS_DONE_PERMISSION),
};
match fs::symlink_metadata(&target) {
Ok(md) if md.file_type().is_symlink() => match policy {
SymlinkPolicy::Refuse => Err(FS_DONE_PERMISSION),
SymlinkPolicy::Operate => Ok(target),
SymlinkPolicy::Follow => {
let resolved = fs::canonicalize(&target).map_err(|e| write_io_status(&e))?;
if resolved.starts_with(root) {
Ok(resolved)
} else {
Err(FS_DONE_PERMISSION)
}
}
},
_ => Ok(target),
}
}
fn temp_sibling(target: &Path) -> PathBuf {
static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let n = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let dir = target.parent().unwrap_or_else(|| Path::new("."));
dir.join(format!(".blit-tmp-{}-{n}", std::process::id()))
}
#[cfg(unix)]
fn apply_mode(f: &fs::File, at: &Path, mode: u32) {
if mode == 0
&& let Ok(md) = fs::metadata(at)
{
let _ = f.set_permissions(md.permissions());
}
}
#[cfg(not(unix))]
fn apply_mode(_f: &fs::File, _at: &Path, _mode: u32) {}
fn fsync_durable(f: &fs::File, target: &Path) -> io::Result<()> {
f.sync_all()?;
#[cfg(unix)]
if let Some(dir) = target.parent()
&& let Ok(d) = fs::File::open(dir)
{
let _ = d.sync_all();
}
let _ = target;
Ok(())
}
fn write_atomic(target: &Path, bytes: &[u8], mode: u32, durable: bool) -> io::Result<()> {
use std::io::Write as _;
let tmp = temp_sibling(target);
let mut opts = fs::OpenOptions::new();
opts.write(true).create_new(true);
#[cfg(unix)]
if mode != 0 {
use std::os::unix::fs::OpenOptionsExt;
opts.mode(mode);
}
let mut f = opts.open(&tmp)?;
let staged = (|| {
f.write_all(bytes)?;
apply_mode(&f, target, mode);
if durable {
f.sync_all()?;
}
Ok(())
})();
drop(f);
if let Err(e) = staged {
let _ = fs::remove_file(&tmp);
return Err(e);
}
if let Err(e) = fs::rename(&tmp, target) {
let _ = fs::remove_file(&tmp);
return Err(e);
}
#[cfg(unix)]
if durable && let Ok(d) = fs::File::open(target.parent().unwrap_or_else(|| Path::new("."))) {
let _ = d.sync_all();
}
Ok(())
}
fn create_exclusive(target: &Path, bytes: &[u8], mode: u32, durable: bool) -> io::Result<()> {
use std::io::Write as _;
let mut opts = fs::OpenOptions::new();
opts.write(true).create_new(true);
#[cfg(unix)]
if mode != 0 {
use std::os::unix::fs::OpenOptionsExt;
opts.mode(mode);
}
let mut f = opts.open(target)?;
let staged = (|| {
f.write_all(bytes)?;
if durable {
fsync_durable(&f, target)?;
}
Ok(())
})();
drop(f);
if let Err(e) = staged {
let _ = fs::remove_file(target);
return Err(e);
}
Ok(())
}
fn current_hash(path: &Path) -> u128 {
match fs::symlink_metadata(path) {
Ok(md) if md.file_type().is_symlink() => match link_target_bytes(path) {
Ok(bytes) => blake3_128(&bytes),
Err(_) => 0,
},
_ => hash_file_streamed(path).unwrap_or(0),
}
}
fn hash_file_streamed(path: &Path) -> io::Result<u128> {
use std::io::Read as _;
let mut f = fs::File::open(path)?;
let mut hasher = blake3::Hasher::new();
let mut buf = [0u8; 64 * 1024];
loop {
let n = f.read(&mut buf)?;
if n == 0 {
break;
}
hasher.update(&buf[..n]);
}
Ok(u128::from_le_bytes(
hasher.finalize().as_bytes()[..16].try_into().unwrap(),
))
}
fn link_target_bytes(path: &Path) -> io::Result<Vec<u8>> {
let target = fs::read_link(path)?;
#[cfg(unix)]
{
use std::os::unix::ffi::OsStrExt;
Ok(target.as_os_str().as_bytes().to_vec())
}
#[cfg(not(unix))]
Ok(target.to_string_lossy().into_owned().into_bytes())
}
#[cfg(unix)]
fn symlink_at(target: &str, at: &Path) -> io::Result<()> {
std::os::unix::fs::symlink(target, at)
}
#[cfg(windows)]
fn symlink_at(target: &str, at: &Path) -> io::Result<()> {
let resolved = at.parent().unwrap_or_else(|| Path::new(".")).join(target);
if resolved.is_dir() {
std::os::windows::fs::symlink_dir(target, at)
} else {
std::os::windows::fs::symlink_file(target, at)
}
}
#[cfg(not(any(unix, windows)))]
fn symlink_at(_target: &str, _at: &Path) -> io::Result<()> {
Err(io::Error::from(io::ErrorKind::Unsupported))
}
fn wire_key_for(root: &Path, abs: &Path) -> Option<String> {
let rel = abs.strip_prefix(root).ok()?;
let mut wire = String::new();
for comp in rel.components() {
wire = join_wire(&wire, &os_to_wire(comp.as_os_str()));
}
Some(wire)
}
fn path_write_lock(path: &Path) -> Arc<Mutex<()>> {
static LOCKS: OnceLock<Mutex<std::collections::HashMap<PathBuf, std::sync::Weak<Mutex<()>>>>> =
OnceLock::new();
let mut map = LOCKS.get_or_init(Default::default).lock().unwrap();
if let Some(existing) = map.get(path).and_then(std::sync::Weak::upgrade) {
return existing;
}
map.retain(|_, w| w.strong_count() > 0);
let lock = Arc::new(Mutex::new(()));
map.insert(path.to_path_buf(), Arc::downgrade(&lock));
lock
}
fn create_parents_confined(root: &Path, target_parent: &Path) -> Result<(), u8> {
let mut existing = target_parent.to_path_buf();
let mut tail: Vec<std::ffi::OsString> = Vec::new();
while !existing.exists() {
let Some(name) = existing.file_name().map(|n| n.to_os_string()) else {
return Err(FS_DONE_INVALID);
};
tail.push(name);
existing = existing.parent().map(Path::to_path_buf).unwrap_or_default();
if existing.as_os_str().is_empty() {
return Err(FS_DONE_INVALID);
}
}
let mut cur = fs::canonicalize(&existing).map_err(|e| write_io_status(&e))?;
if !cur.starts_with(root) {
return Err(FS_DONE_PERMISSION);
}
for name in tail.iter().rev() {
cur.push(name);
if let Err(e) = fs::create_dir(&cur) {
let real_dir = fs::symlink_metadata(&cur)
.map(|m| m.file_type().is_dir())
.unwrap_or(false);
if !real_dir {
return Err(write_io_status(&e));
}
}
match fs::canonicalize(&cur) {
Ok(c) if c.starts_with(root) => cur = c,
Ok(_) => return Err(FS_DONE_PERMISSION),
Err(e) => return Err(write_io_status(&e)),
}
}
Ok(())
}
pub struct BlobStore {
budget: usize,
total: usize,
seq: u64,
by_hash: std::collections::HashMap<u128, (Arc<Vec<u8>>, u64)>,
by_age: BTreeMap<u64, u128>,
}
impl BlobStore {
pub fn new(budget: usize) -> Self {
BlobStore {
budget,
total: 0,
seq: 0,
by_hash: Default::default(),
by_age: Default::default(),
}
}
pub fn get(&mut self, hash: u128) -> Option<Arc<Vec<u8>>> {
let (data, seq) = self.by_hash.get(&hash)?.clone();
self.by_age.remove(&seq);
self.seq += 1;
self.by_age.insert(self.seq, hash);
self.by_hash.insert(hash, (data.clone(), self.seq));
Some(data)
}
pub fn put(&mut self, hash: u128, data: Arc<Vec<u8>>) {
if data.len() > self.budget {
return;
}
if self.by_hash.contains_key(&hash) {
self.get(hash);
return;
}
self.seq += 1;
self.total += data.len();
self.by_age.insert(self.seq, hash);
self.by_hash.insert(hash, (data, self.seq));
while self.total > self.budget {
let (&seq, &oldest) = self
.by_age
.iter()
.next()
.expect("total > 0 implies entries");
self.by_age.remove(&seq);
if let Some((old, _)) = self.by_hash.remove(&oldest) {
self.total -= old.len();
}
}
}
}
pub fn blob_store() -> &'static Mutex<BlobStore> {
static STORE: OnceLock<Mutex<BlobStore>> = OnceLock::new();
STORE.get_or_init(|| {
Mutex::new(BlobStore::new(
env_u64("BLIT_FS_BLOB_MAX", 256 * 1024 * 1024) as usize,
))
})
}
fn push_leb128(out: &mut Vec<u8>, mut value: u64) {
loop {
let byte = (value & 0x7F) as u8;
value >>= 7;
if value == 0 {
out.push(byte);
return;
}
out.push(byte | 0x80);
}
}
pub fn encode_delta(base: &[u8], new: &[u8]) -> Vec<u8> {
let bound = base.len().min(new.len());
let mut prefix = 0;
while prefix < bound && base[prefix] == new[prefix] {
prefix += 1;
}
let mut suffix = 0;
let bound = bound - prefix;
while suffix < bound && base[base.len() - 1 - suffix] == new[new.len() - 1 - suffix] {
suffix += 1;
}
let mut ops = Vec::new();
if prefix > 0 {
ops.push(0x01);
push_leb128(&mut ops, 0);
push_leb128(&mut ops, prefix as u64);
}
let middle = &new[prefix..new.len() - suffix];
if !middle.is_empty() {
ops.push(0x02);
push_leb128(&mut ops, middle.len() as u64);
ops.extend_from_slice(middle);
}
if suffix > 0 {
ops.push(0x01);
push_leb128(&mut ops, (base.len() - suffix) as u64);
push_leb128(&mut ops, suffix as u64);
}
ops
}
struct Reconciler {
root: PathBuf,
single: bool,
opts: SyncOptions,
ignores: Option<ignores::Ignores>,
rx: Receiver<RootMsg>,
backend: Box<dyn BackendHandle>,
canonical: Index,
snapshot: Arc<Index>,
subs: std::collections::HashMap<u64, (Sender<SyncMsg>, Duration)>,
latency: Duration,
dirty: std::collections::BTreeSet<String>,
changed: std::collections::BTreeSet<String>,
recheck: std::collections::BTreeSet<String>,
full_rescan: bool,
pending_since: Option<Instant>,
hash_dirty_since: Option<Instant>,
closed: Option<u8>,
closed_flag: Arc<OnceLock<u8>>,
}
const HASH_PUBLISH_INTERVAL: Duration = Duration::from_millis(500);
fn record_merge_changed(
old: &Index,
new: &Index,
changed: &mut std::collections::BTreeSet<String>,
) {
let mut oi = old.iter().peekable();
let mut ni = new.iter().peekable();
loop {
match (oi.peek(), ni.peek()) {
(Some((ok, ov)), Some((nk, nv))) => {
if ok == nk {
if ov != nv {
changed.insert((*ok).clone());
}
oi.next();
ni.next();
} else if ok < nk {
changed.insert((*ok).clone());
oi.next();
} else {
changed.insert((*nk).clone());
ni.next();
}
}
(Some((ok, _)), None) => {
changed.insert((*ok).clone());
oi.next();
}
(None, Some((nk, _))) => {
changed.insert((*nk).clone());
ni.next();
}
(None, None) => break,
}
}
}
impl Reconciler {
fn new(
key: RootKey,
single: bool,
rx: Receiver<RootMsg>,
backend: Box<dyn BackendHandle>,
closed_flag: Arc<OnceLock<u8>>,
) -> Self {
let opts = SyncOptions {
recursive: key.recursive,
cross_filesystem: key.cross_filesystem,
..Default::default()
};
let ignores = (!single && !key.ignores.is_empty())
.then(|| ignores::Ignores::new(&key.path, &key.ignores));
Reconciler {
root: key.path,
single,
latency: opts.latency,
opts,
ignores,
rx,
backend,
canonical: Index::new(),
snapshot: Arc::new(Index::new()),
subs: Default::default(),
dirty: Default::default(),
changed: Default::default(),
recheck: Default::default(),
full_rescan: false,
pending_since: None,
hash_dirty_since: None,
closed: None,
closed_flag,
}
}
fn run(mut self) {
if let Some(ignores) = &self.ignores {
for dir in ignores.external_watch_dirs() {
self.backend.watch_outside(&dir);
}
}
match self.scan_all() {
Ok(index) => {
self.canonical = index;
self.snapshot = Arc::new(self.canonical.clone());
}
Err(reason) => self.close(reason),
}
loop {
let deadline = |since: Option<Instant>, window: Duration| {
since.map(|s| (s + window).saturating_duration_since(Instant::now()))
};
let timeout = if self.closed.is_some() {
Duration::from_secs(3600)
} else {
[
deadline(self.pending_since, self.latency),
deadline(self.hash_dirty_since, HASH_PUBLISH_INTERVAL),
]
.into_iter()
.flatten()
.min()
.unwrap_or(Duration::from_secs(3600))
};
match self.rx.recv_timeout(timeout) {
Ok(RootMsg::Hint(hint)) => self.note_hint(hint),
Ok(RootMsg::Subscribe { id, tx, latency }) => {
let update = match self.closed {
Some(reason) => RootUpdate::Closed(reason),
None => RootUpdate::Snapshot {
index: self.snapshot.clone(),
settled: None,
changed: None,
recheck: Default::default(),
},
};
let _ = tx.send(SyncMsg::Root(update));
self.subs.insert(id, (tx, latency));
self.recompute_latency();
}
Ok(RootMsg::Unsubscribe { id }) => {
self.subs.remove(&id);
self.recompute_latency();
}
Ok(RootMsg::HashLearned { path, meta }) => {
if let Some(existing) = self.canonical.get_mut(&path)
&& existing.hash != meta.hash
&& existing.node_type == meta.node_type
&& existing.dev_ino == meta.dev_ino
&& existing.size == meta.size
&& existing.mtime_ns == meta.mtime_ns
{
existing.hash = meta.hash;
self.changed.insert(path);
if self.hash_dirty_since.is_none() {
self.hash_dirty_since = Some(Instant::now());
}
}
}
Err(RecvTimeoutError::Timeout) => {}
Err(RecvTimeoutError::Disconnected) => return,
}
let elapsed = |since: Option<Instant>, window: Duration| {
since.is_some_and(|s| Instant::now().saturating_duration_since(s) >= window)
};
if self.closed.is_none()
&& (elapsed(self.pending_since, self.latency)
|| elapsed(self.hash_dirty_since, HASH_PUBLISH_INTERVAL))
{
self.tick();
}
}
}
fn ignored(&mut self, rel: &str, is_dir: bool) -> bool {
match &mut self.ignores {
Some(ignores) => ignores.matched(rel, is_dir),
None => false,
}
}
fn ignore_rules_changed(&mut self, abs: &Path, rel: &str) -> bool {
match &mut self.ignores {
Some(ignores) => ignores.source_affects_rules(abs, rel),
None => false,
}
}
fn retain_watched_dirs(&self) {
let root = &self.root;
let canonical = &self.canonical;
self.backend.retain_dirs(&|abs| {
wire_key_for(root, abs)
.and_then(|key| canonical.get(&key).map(|m| m.node_type == FS_ENTRY_DIR))
.unwrap_or(false)
});
}
fn recompute_latency(&mut self) {
self.latency = self
.subs
.values()
.map(|(_, latency)| *latency)
.min()
.unwrap_or(self.opts.latency);
}
fn close(&mut self, reason: u8) {
self.closed = Some(reason);
let _ = self.closed_flag.set(reason);
self.pending_since = None;
for (tx, _) in self.subs.values() {
let _ = tx.send(SyncMsg::Root(RootUpdate::Closed(reason)));
}
}
fn note_hint(&mut self, hint: Hint) {
if self.single {
let relevant = match hint {
Hint::Rescan => true,
Hint::Dirty(abs) => abs == self.root || Some(abs.as_path()) == self.root.parent(),
};
if relevant {
self.dirty.insert(String::new());
if self.pending_since.is_none() {
self.pending_since = Some(Instant::now());
}
}
return;
}
match hint {
Hint::Rescan => self.full_rescan = true,
Hint::Dirty(abs) => {
let rel = match abs.strip_prefix(&self.root) {
Ok(rel) => rel,
Err(_) => {
if self
.ignores
.as_ref()
.is_some_and(|i| i.is_external_source(&abs))
{
if let Some(ignores) = &mut self.ignores {
ignores.invalidate();
}
self.full_rescan = true;
if self.pending_since.is_none() {
self.pending_since = Some(Instant::now());
}
}
return;
}
};
let mut wire = String::new();
let mut depth = 0usize;
for comp in rel.components() {
wire = join_wire(&wire, &os_to_wire(comp.as_os_str()));
depth += 1;
}
if !self.opts.recursive && depth > 1 {
return;
}
if self.ignore_rules_changed(&abs, &wire) {
if let Some(ignores) = &mut self.ignores {
ignores.invalidate();
}
self.full_rescan = true;
} else if self.ignored(&wire, false) {
if let Some(parent) = parent_wire(&wire)
&& self.canonical.get(parent).is_some_and(|m| !m.filtered)
{
self.dirty.insert(parent.to_string());
if self.pending_since.is_none() {
self.pending_since = Some(Instant::now());
}
}
return;
}
self.dirty.insert(wire);
}
}
if self.pending_since.is_none() {
self.pending_since = Some(Instant::now());
}
}
fn tick(&mut self) {
let settled = self.pending_since;
self.pending_since = None;
self.hash_dirty_since = None;
if self.full_rescan {
self.full_rescan = false;
self.dirty.clear();
match self.scan_all() {
Ok(index) => {
record_merge_changed(&self.canonical, &index, &mut self.changed);
self.canonical = index;
self.retain_watched_dirs();
if let Some(ignores) = &self.ignores {
for dir in ignores.external_watch_dirs() {
self.backend.watch_outside(&dir);
}
}
}
Err(reason) => return self.close(reason),
}
} else {
let dirty = std::mem::take(&mut self.dirty);
for rel in dirty {
if let Err(reason) = self.reconcile(&rel) {
return self.close(reason);
}
}
}
let prev_snapshot = self.snapshot.clone();
self.changed
.retain(|k| self.canonical.get(k) != prev_snapshot.get(k));
if !self.changed.is_empty() || !self.recheck.is_empty() {
let changed = Arc::new(std::mem::take(&mut self.changed));
let recheck = Arc::new(std::mem::take(&mut self.recheck));
if !changed.is_empty() {
self.snapshot = Arc::new(self.canonical.clone());
}
for (tx, _) in self.subs.values() {
let _ = tx.send(SyncMsg::Root(RootUpdate::Snapshot {
index: self.snapshot.clone(),
settled,
changed: Some(changed.clone()),
recheck: recheck.clone(),
}));
}
}
}
fn scan_all(&mut self) -> Result<Index, u8> {
if self.single {
return self.scan_single();
}
let mut index = Index::new();
let root = self.root.clone();
self.scan_into(&mut index, &root, "", self.opts.recursive, None)
.map_err(|e| match e.kind() {
io::ErrorKind::NotFound => FS_CLOSED_ROOT_GONE,
io::ErrorKind::PermissionDenied => FS_CLOSED_PERMISSION_LOST_COMPAT,
_ if e.raw_os_error() == Some(RESOURCE_LIMIT_ERRNO) => FS_CLOSED_RESOURCE_LIMIT,
_ => FS_CLOSED_RESOURCE_LIMIT,
})?;
Ok(index)
}
fn scan_single(&self) -> Result<Index, u8> {
let mut index = Index::new();
match stat_meta(&self.root) {
Ok(meta) => {
index.insert(String::new(), meta);
}
Err(e) => {
if !self.root.parent().map(Path::exists).unwrap_or(false) {
return Err(FS_CLOSED_ROOT_GONE);
}
if e.kind() == io::ErrorKind::PermissionDenied {
return Err(FS_CLOSED_PERMISSION_LOST_COMPAT);
}
}
}
Ok(index)
}
fn reconcile_single(&mut self) -> Result<(), u8> {
match stat_meta(&self.root) {
Ok(meta) => {
let preserved = self
.canonical
.get("")
.and_then(|m| (!m.content_changed(&meta)).then_some(m.hash));
let mut meta = meta;
if let Some(h) = preserved {
meta.hash = h;
self.note_racy("", &meta);
}
self.index_insert(String::new(), meta);
}
Err(e) => {
if !self.root.parent().map(Path::exists).unwrap_or(false) {
return Err(FS_CLOSED_ROOT_GONE);
}
if e.kind() == io::ErrorKind::PermissionDenied {
return Err(FS_CLOSED_PERMISSION_LOST_COMPAT);
}
self.index_remove("");
}
}
Ok(())
}
fn scan_into(
&mut self,
index: &mut Index,
abs: &Path,
rel: &str,
recurse: bool,
root_dev: Option<u64>,
) -> io::Result<()> {
let mut ancestors = Vec::new();
self.scan_into_inner(index, abs, rel, recurse, root_dev, &mut ancestors)?;
Ok(())
}
fn scan_into_inner(
&mut self,
index: &mut Index,
abs: &Path,
rel: &str,
recurse: bool,
root_dev: Option<u64>,
ancestors: &mut Vec<(u64, u64)>,
) -> io::Result<bool> {
let meta = stat_meta(abs)?;
if !rel.is_empty() && self.ignored(rel, meta.enumerable_dir()) {
return Ok(true);
}
if index.len() >= self.opts.max_entries {
return Err(io::Error::from_raw_os_error(RESOURCE_LIMIT_ERRNO));
}
let node_type = meta.node_type;
let link_dir = meta.link_dir;
let self_id = meta.dev_ino;
let dev = meta.dev_ino.0;
index.insert(rel.to_string(), meta);
let (descend_id, dev) = if node_type == FS_ENTRY_SYMLINK {
if !link_dir {
return Ok(false); }
let Ok(target) = fs::metadata(abs) else {
return Ok(false);
};
let id = target_identity(&target);
if id == (0, 0) {
return Ok(false);
}
if ancestors.contains(&id) {
return Ok(false); }
(id, id.0)
} else if node_type == FS_ENTRY_DIR {
(self_id, dev)
} else {
return Ok(false);
};
ancestors.push(descend_id);
let real_dir = node_type == FS_ENTRY_DIR;
let result =
self.scan_children(index, abs, rel, recurse, root_dev, dev, real_dir, ancestors);
ancestors.pop();
result.map(|()| false)
}
#[allow(clippy::too_many_arguments)]
fn scan_children(
&mut self,
index: &mut Index,
abs: &Path,
rel: &str,
recurse: bool,
root_dev: Option<u64>,
dev: u64,
real_dir: bool,
ancestors: &mut Vec<(u64, u64)>,
) -> io::Result<()> {
let root_dev = root_dev.or(Some(dev));
if !self.opts.cross_filesystem && Some(dev) != root_dev {
return Ok(()); }
if real_dir && !self.backend.add_dir(abs) {
return Err(io::Error::from_raw_os_error(RESOURCE_LIMIT_ERRNO));
}
if !recurse && !rel.is_empty() {
return Ok(());
}
let entries = match fs::read_dir(abs) {
Ok(e) => e,
Err(_) => return Ok(()), };
let mut filtered = false;
for entry in entries.flatten() {
let name = os_to_wire(&entry.file_name());
let child_rel = join_wire(rel, &name);
let child_abs = entry.path();
let child_recurse = self.opts.recursive;
match self.scan_into_inner(
index,
&child_abs,
&child_rel,
child_recurse,
root_dev,
ancestors,
) {
Ok(excluded) => filtered |= excluded,
Err(e) if e.raw_os_error() == Some(RESOURCE_LIMIT_ERRNO) => return Err(e),
Err(_) => {}
}
}
if filtered && let Some(dir) = index.get_mut(rel) {
dir.filtered = true;
}
Ok(())
}
fn note_racy(&mut self, key: &str, meta: &NodeMeta) {
if matches!(meta.node_type, FS_ENTRY_FILE | FS_ENTRY_SYMLINK) && racily_clean(meta.mtime_ns)
{
self.recheck.insert(key.to_string());
}
}
fn index_insert(&mut self, key: String, meta: NodeMeta) {
if self.canonical.get(&key) != Some(&meta) {
self.changed.insert(key.clone());
self.canonical.insert(key, meta);
}
}
fn index_remove(&mut self, key: &str) {
let Some(meta) = self.canonical.remove(key) else {
return;
};
self.changed.insert(key.to_string());
if meta.node_type == FS_ENTRY_DIR
&& let Some(abs) = resolve_wire_path(&self.root, key)
{
self.backend.remove_dir(&abs);
}
}
fn remove_index_subtree(&mut self, rel: &str, keep_root: bool) {
for key in subtree_keys(&self.canonical, rel) {
if keep_root && key == rel {
continue;
}
self.index_remove(&key);
}
}
fn root_device(&self) -> Option<u64> {
self.canonical.get("").map(|m| m.dev_ino.0)
}
fn reconcile(&mut self, rel: &str) -> Result<(), u8> {
if self.single {
return self.reconcile_single();
}
let Some(abs) = resolve_wire_path(&self.root, rel) else {
return Ok(());
};
match stat_meta(&abs) {
Err(_) => {
if rel.is_empty() {
return Err(FS_CLOSED_ROOT_GONE);
}
self.remove_index_subtree(rel, false);
}
Ok(meta) => {
if self.ignored(rel, meta.enumerable_dir()) {
self.remove_index_subtree(rel, false);
if let Some(parent) = parent_wire(rel)
&& let Some(meta) = self.canonical.get(parent)
&& !meta.filtered
{
let mut meta = meta.clone();
meta.filtered = true;
self.index_insert(parent.to_string(), meta);
}
return Ok(());
}
if !self.opts.cross_filesystem
&& !rel.is_empty()
&& let Some(root_dev) = self.canonical.get("").map(|m| m.dev_ino.0)
&& meta.dev_ino.0 != root_dev
{
let parent_on_root = parent_wire(rel)
.and_then(|p| self.canonical.get(p))
.is_some_and(|m| m.dev_ino.0 == root_dev);
if parent_on_root {
self.index_insert(rel.to_string(), meta);
self.check_budget()?;
} else {
self.remove_index_subtree(rel, false);
}
return Ok(());
}
let known = self.canonical.contains_key(rel);
let was_dir = self
.canonical
.get(rel)
.map(|m| m.enumerable_dir())
.unwrap_or(false);
let is_dir = meta.enumerable_dir();
let preserved_hash = self
.canonical
.get(rel)
.and_then(|m| (!m.content_changed(&meta)).then_some(m.hash));
let was_filtered = self.canonical.get(rel).is_some_and(|m| m.filtered);
let mut meta = meta;
meta.filtered = was_filtered;
if let Some(h) = preserved_hash {
meta.hash = h;
self.note_racy(rel, &meta);
}
self.index_insert(rel.to_string(), meta);
self.check_budget()?;
if is_dir && (!known || !was_dir) {
let mut sub = Index::new();
let bound = self.root_device();
match self.scan_into(&mut sub, &abs, rel, self.opts.recursive, bound) {
Ok(()) => {}
Err(e) if e.raw_os_error() == Some(RESOURCE_LIMIT_ERRNO) => {
return Err(FS_CLOSED_RESOURCE_LIMIT);
}
Err(_) => {}
}
for (k, v) in sub {
self.index_insert(k, v);
}
self.check_budget()?;
} else if is_dir && self.opts.recursive {
self.reconcile_children(&abs, rel)?;
}
if was_dir && !is_dir {
self.remove_index_subtree(rel, true);
}
}
}
Ok(())
}
fn check_budget(&self) -> Result<(), u8> {
if self.canonical.len() > self.opts.max_entries {
Err(FS_CLOSED_RESOURCE_LIMIT)
} else {
Ok(())
}
}
fn reconcile_children(&mut self, abs: &Path, rel: &str) -> Result<(), u8> {
let Ok(entries) = fs::read_dir(abs) else {
return Ok(());
};
let mut seen: std::collections::HashSet<String> = Default::default();
let mut new_dirs: Vec<(PathBuf, String)> = Vec::new();
let mut filtered = false;
for entry in entries.flatten() {
let name = os_to_wire(&entry.file_name());
let child_rel = join_wire(rel, &name);
if let Ok(meta) = stat_meta(&entry.path()) {
if self.ignored(&child_rel, meta.enumerable_dir()) {
filtered = true;
continue;
}
let newly_dir = meta.enumerable_dir()
&& self
.canonical
.get(&child_rel)
.map(|m| !m.enumerable_dir())
.unwrap_or(true);
let preserved = self
.canonical
.get(&child_rel)
.and_then(|m| (!m.content_changed(&meta)).then_some(m.hash));
let mut meta = meta;
meta.filtered = self.canonical.get(&child_rel).is_some_and(|m| m.filtered);
if let Some(h) = preserved {
meta.hash = h;
self.note_racy(&child_rel, &meta);
}
if newly_dir {
new_dirs.push((entry.path(), child_rel.clone()));
}
self.index_insert(child_rel.clone(), meta);
self.check_budget()?;
}
seen.insert(child_rel);
}
if let Some(dir) = self.canonical.get(rel)
&& dir.filtered != filtered
{
let mut meta = dir.clone();
meta.filtered = filtered;
self.index_insert(rel.to_string(), meta);
}
let prefix = if rel.is_empty() {
String::new()
} else {
format!("{rel}/")
};
let gone: Vec<String> = self
.canonical
.range(prefix.clone()..)
.take_while(|(k, _)| k.starts_with(&prefix))
.filter(|(k, _)| {
k.as_str() != rel && {
let rest = &k[prefix.len()..];
let child_end = prefix.len() + rest.find('/').unwrap_or(rest.len());
!seen.contains(&k[..child_end])
}
})
.map(|(k, _)| k.clone())
.collect();
for k in gone {
self.index_remove(&k);
}
let bound = self.root_device();
for (abs, rel) in new_dirs {
let mut sub = Index::new();
match self.scan_into(&mut sub, &abs, &rel, self.opts.recursive, bound) {
Ok(()) => {}
Err(e) if e.raw_os_error() == Some(RESOURCE_LIMIT_ERRNO) => {
return Err(FS_CLOSED_RESOURCE_LIMIT);
}
Err(_) => {}
}
for (k, v) in sub {
self.index_insert(k, v);
}
self.check_budget()?;
}
Ok(())
}
}
enum Exit {
ClientGone,
Closed(u8),
Stopped,
}
enum ContentRead {
Stable { hash: u128, data: Arc<Vec<u8>> },
Unstable,
Unreadable,
}
struct RetryEntry {
failures: u32,
due: Instant,
}
fn retry_backoff(failures: u32, latency: Duration) -> Duration {
const RETRY_BACKOFF_CAP: Duration = Duration::from_secs(2);
latency
.saturating_mul(
1u32.checked_shl(failures.saturating_sub(1))
.unwrap_or(u32::MAX),
)
.min(RETRY_BACKOFF_CAP)
}
struct SyncEngine {
sync_id: u16,
root: PathBuf,
single: bool,
opts: SyncOptions,
rx: Receiver<SyncMsg>,
outbox: Outbox,
shared: Arc<SharedRootHandle>,
sub_id: u64,
latest: Arc<Index>,
snapshot_dirty: bool,
shadow: Arc<Index>,
pending_since: Option<Instant>,
next_update_id: u32,
highest_sent: u32,
unacked: std::collections::VecDeque<(u32, usize)>,
unacked_bytes: usize,
initial_sent: bool,
held: std::collections::HashMap<String, u128>,
retry: BTreeMap<String, RetryEntry>,
pending_changed: std::collections::BTreeSet<String>,
pending_recheck: std::collections::BTreeSet<String>,
full_diff: bool,
}
impl SyncEngine {
fn new(
sync_id: u16,
shared: Arc<SharedRootHandle>,
sub_id: u64,
opts: SyncOptions,
rx: Receiver<SyncMsg>,
outbox: Outbox,
) -> Self {
SyncEngine {
sync_id,
root: shared.key.path.clone(),
single: shared.single,
opts,
rx,
outbox,
shared,
sub_id,
latest: Arc::new(Index::new()),
snapshot_dirty: false,
shadow: Arc::new(Index::new()),
pending_since: None,
next_update_id: 1,
highest_sent: 0,
unacked: Default::default(),
unacked_bytes: 0,
initial_sent: false,
held: Default::default(),
retry: Default::default(),
pending_changed: Default::default(),
pending_recheck: Default::default(),
full_diff: false,
}
}
fn run(mut self) {
let exit = self.event_loop();
let _ = self
.shared
.tx
.send(RootMsg::Unsubscribe { id: self.sub_id });
match exit {
Exit::ClientGone => {}
Exit::Stopped => {
self.drain_pending_commands();
let _ = (self.outbox)(msg_fs_closed(self.sync_id, FS_CLOSED_CLIENT_REQUEST));
}
Exit::Closed(reason) => {
self.drain_pending_commands();
let _ = (self.outbox)(msg_fs_closed(self.sync_id, reason));
}
}
}
fn drain_pending_commands(&mut self) {
while let Ok(msg) = self.rx.try_recv() {
match msg {
SyncMsg::Cmd(Command::Write(w)) => {
let _ = (self.outbox)(msg_fs_done(w.nonce, FS_DONE_OTHER, 0, 0));
}
SyncMsg::Cmd(Command::Op(o)) => {
let _ = (self.outbox)(msg_fs_done(o.nonce, FS_DONE_OTHER, 0, 0));
}
SyncMsg::Cmd(Command::Fetch { nonce, .. }) => {
let _ = (self.outbox)(msg_fs_file(nonce, blit_remote::fs::FS_FILE_OTHER, &[]));
}
SyncMsg::Cmd(Command::Ack(_) | Command::Stop) | SyncMsg::Root(_) => {}
}
}
}
fn event_loop(&mut self) -> Exit {
loop {
let timeout = match self.pending_since {
Some(since) if self.unacked_bytes < self.opts.window_bytes => {
(since + self.opts.latency).saturating_duration_since(Instant::now())
}
_ => Duration::from_secs(3600),
};
match self.rx.recv_timeout(timeout) {
Ok(SyncMsg::Root(update)) => {
if let Err(exit) = self.handle_root(update) {
return exit;
}
}
Ok(SyncMsg::Cmd(Command::Ack(update_id))) => {
if let Err(exit) = self.handle_ack(update_id) {
return exit;
}
}
Ok(SyncMsg::Cmd(Command::Fetch { nonce, path })) => {
if !self.handle_fetch(nonce, &path) {
return Exit::ClientGone;
}
}
Ok(SyncMsg::Cmd(Command::Write(w))) => {
if !self.handle_write(w) {
return Exit::ClientGone;
}
}
Ok(SyncMsg::Cmd(Command::Op(o))) => {
if !self.handle_op(o) {
return Exit::ClientGone;
}
}
Ok(SyncMsg::Cmd(Command::Stop)) => return Exit::Stopped,
Err(RecvTimeoutError::Timeout) => {}
Err(RecvTimeoutError::Disconnected) => return Exit::ClientGone,
}
if let Some(since) = self.pending_since
&& Instant::now().saturating_duration_since(since) >= self.opts.latency
&& self.unacked_bytes < self.opts.window_bytes
&& let Err(exit) = self.tick()
{
return exit;
}
}
}
fn handle_root(&mut self, update: RootUpdate) -> Result<(), Exit> {
match update {
RootUpdate::Snapshot {
index,
settled,
changed,
recheck,
} => {
self.latest = index;
self.snapshot_dirty = true;
self.pending_recheck.extend(recheck.iter().cloned());
match changed {
Some(set) if !self.full_diff => {
self.pending_changed.extend(set.iter().cloned());
}
Some(_) => {}
None => {
self.full_diff = true;
self.pending_changed.clear();
}
}
let due = settled.unwrap_or_else(|| {
Instant::now()
.checked_sub(self.opts.latency)
.unwrap_or_else(Instant::now)
});
self.pending_since = Some(match self.pending_since {
Some(existing) if existing <= due => existing,
_ => due,
});
Ok(())
}
RootUpdate::Closed(reason) => Err(Exit::Closed(reason)),
}
}
fn handle_ack(&mut self, update_id: u32) -> Result<(), Exit> {
let ahead = update_id.wrapping_sub(self.highest_sent);
if ahead != 0 && ahead < 0x8000_0000 {
return Err(Exit::Closed(FS_CLOSED_BACKEND_FAILED_COMPAT));
}
while let Some(&(id, bytes)) = self.unacked.front() {
if update_id.wrapping_sub(id) < 0x8000_0000 {
self.unacked.pop_front();
self.unacked_bytes -= bytes;
} else {
break;
}
}
Ok(())
}
fn tick(&mut self) -> Result<(), Exit> {
self.pending_since = None;
if self.initial_sent && !self.snapshot_dirty && self.retry.is_empty() {
return Ok(());
}
let canonical = self.latest.clone();
let initial = !self.initial_sent;
self.snapshot_dirty = false;
let full = std::mem::take(&mut self.full_diff);
let changed = std::mem::take(&mut self.pending_changed);
let recheck = std::mem::take(&mut self.pending_recheck);
self.emit_updates(&canonical, initial, full, &changed, &recheck)?;
self.shadow = canonical;
self.initial_sent = true;
if self.snapshot_dirty {
self.pending_since = Some(Instant::now());
} else if let Some(due) = self.retry.values().map(|e| e.due).min() {
self.pending_since = Some(
due.checked_sub(self.opts.latency)
.unwrap_or_else(Instant::now),
);
}
Ok(())
}
fn emit_updates(
&mut self,
canonical: &Arc<Index>,
initial: bool,
full: bool,
changed: &std::collections::BTreeSet<String>,
recheck: &std::collections::BTreeSet<String>,
) -> Result<(), Exit> {
if initial {
return self.emit_initial(canonical);
}
let mut ops = if full {
diff(&self.shadow, canonical)
} else {
diff_changed(&self.shadow, canonical, changed)
};
for op in &ops {
if let DiffOp::Move { from, to } = op {
self.rekey_move(from, to);
}
}
self.retry.retain(|path, _| canonical.contains_key(path));
let now = Instant::now();
let forced: Vec<String> = self
.retry
.iter()
.filter(|(path, entry)| {
entry.due <= now
&& !ops
.iter()
.any(|op| matches!(op, DiffOp::Upsert { path: p, .. } if p == *path))
})
.map(|(path, _)| path.clone())
.collect();
ops.extend(forced.into_iter().map(|path| DiffOp::Upsert {
path,
content_changed: true,
}));
let racy: Vec<String> = recheck
.iter()
.filter(|path| {
!ops.iter()
.any(|op| matches!(op, DiffOp::Upsert { path: p, .. } if p == *path))
&& self.content_diverged(path, canonical)
})
.cloned()
.collect();
ops.extend(racy.into_iter().map(|path| DiffOp::Upsert {
path,
content_changed: true,
}));
if ops.is_empty() {
return Ok(());
}
let mut buf: Vec<u8> = Vec::new();
let mut reset_pending = false;
for op in &ops {
match op {
DiffOp::Delete { path } => {
self.held.retain(|held_path, _| !is_under(held_path, path));
append_fs_record(&mut buf, &FsRecord::Delete { path });
}
DiffOp::Move { from, to } => {
append_fs_record(&mut buf, &FsRecord::Move { from, to });
}
DiffOp::Upsert {
path,
content_changed,
} => {
if let Some(meta) = canonical.get(path) {
self.append_upsert(&mut buf, path, meta, *content_changed);
}
}
}
if buf.len() >= self.opts.batch_target {
self.send_update(std::mem::take(&mut buf), &mut reset_pending, false)?;
}
}
if !buf.is_empty() {
self.send_update(buf, &mut reset_pending, false)?;
}
Ok(())
}
fn emit_initial(&mut self, canonical: &Arc<Index>) -> Result<(), Exit> {
let mut buf: Vec<u8> = Vec::new();
let mut reset_pending = true;
let index: &Index = canonical;
for (path, meta) in index.iter() {
self.append_upsert(&mut buf, path, meta, true);
if buf.len() >= self.opts.batch_target {
self.send_update(std::mem::take(&mut buf), &mut reset_pending, false)?;
}
}
self.send_update(buf, &mut reset_pending, true)?;
Ok(())
}
fn append_upsert(
&mut self,
buf: &mut Vec<u8>,
path: &str,
meta: &NodeMeta,
content_changed: bool,
) {
let prior_failures = self.retry.remove(path).map(|e| e.failures).unwrap_or(0);
let was_retry = prior_failures > 0;
let mut entry_flags = meta.node_type & FS_ENTRY_TYPE_MASK;
if meta.link_dir {
entry_flags |= FS_ENTRY_LINK_DIR;
}
if meta.filtered {
entry_flags |= FS_ENTRY_FILTERED;
}
let mut hash = meta.hash;
let mut full: Option<Arc<Vec<u8>>> = None;
let mut delta: Option<Vec<u8>> = None;
if matches!(meta.node_type, FS_ENTRY_FILE | FS_ENTRY_SYMLINK) {
let inline_cap = self
.opts
.inline_max
.min(blit_remote::fs::FS_MAX_DECOMPRESSED as u64);
if !self.opts.content || meta.size > inline_cap {
entry_flags |= FS_ENTRY_NO_CONTENT;
self.held.remove(path);
} else if content_changed || meta.hash == 0 {
match self.read_content(path, meta) {
ContentRead::Stable {
hash: read_hash,
data,
} => {
hash = read_hash;
if self.held.get(path) == Some(&hash) {
} else {
delta = self
.held
.get(path)
.and_then(|&base_hash| blob_store().lock().unwrap().get(base_hash))
.map(|base| encode_delta(&base, &data))
.filter(|ops| ops.len() * 8 < data.len() * 7);
if delta.is_none() {
full = Some(data.clone());
}
self.held.insert(path.to_string(), hash);
}
}
ContentRead::Unstable => {
self.held.remove(path);
self.note_retry(path, prior_failures);
if was_retry {
return;
}
entry_flags |= FS_ENTRY_UNSTABLE;
}
ContentRead::Unreadable => {
self.held.remove(path);
self.note_retry(path, prior_failures);
if was_retry {
return;
}
entry_flags |= FS_ENTRY_UNREADABLE;
}
}
}
}
let content = match (&delta, &full) {
(Some(ops), _) => FsContent::Delta(ops),
(None, Some(data)) => FsContent::Full(data.as_slice()),
(None, None) => FsContent::None,
};
append_fs_record(
buf,
&FsRecord::Upsert {
path,
entry_flags,
size: meta.size,
mtime_ns: meta.mtime_ns,
mode: meta.mode,
hash,
content,
},
);
}
fn note_retry(&mut self, path: &str, prior_failures: u32) {
let failures = prior_failures + 1;
self.retry.insert(
path.to_string(),
RetryEntry {
failures,
due: Instant::now() + retry_backoff(failures, self.opts.latency),
},
);
}
fn content_diverged(&self, path: &str, canonical: &Index) -> bool {
if !self.opts.content {
return false;
}
let Some(&held) = self.held.get(path) else {
return false;
};
let Some(meta) = canonical.get(path) else {
return false;
};
if !matches!(meta.node_type, FS_ENTRY_FILE | FS_ENTRY_SYMLINK) {
return false;
}
let Some(abs) = resolve_wire_path(&self.root, path) else {
return false;
};
match read_verified_meta(&abs) {
ReadMetaOutcome::Stable(data, _) => blake3_128(&data) != held,
ReadMetaOutcome::Unstable | ReadMetaOutcome::Unreadable => false,
}
}
fn read_content(&self, path: &str, meta: &NodeMeta) -> ContentRead {
if meta.hash != 0
&& let Some(data) = blob_store().lock().unwrap().get(meta.hash)
{
return ContentRead::Stable {
hash: meta.hash,
data,
};
}
if meta.hash == 0
&& let Some(learned) = self.shared.learned.lock().unwrap().get(path).cloned()
&& learned.hash != 0
&& learned.node_type == meta.node_type
&& learned.dev_ino == meta.dev_ino
&& learned.size == meta.size
&& learned.mtime_ns == meta.mtime_ns
&& let Some(data) = blob_store().lock().unwrap().get(learned.hash)
{
return ContentRead::Stable {
hash: learned.hash,
data,
};
}
let Some(abs) = resolve_wire_path(&self.root, path) else {
return ContentRead::Unreadable;
};
match read_verified_meta(&abs) {
ReadMetaOutcome::Stable(data, mut stat) => {
let hash = blake3_128(&data);
let data = Arc::new(data);
blob_store().lock().unwrap().put(hash, data.clone());
stat.hash = hash;
if !racily_clean(stat.mtime_ns) {
self.teach_hash(path, stat);
}
ContentRead::Stable { hash, data }
}
ReadMetaOutcome::Unstable => ContentRead::Unstable,
ReadMetaOutcome::Unreadable => ContentRead::Unreadable,
}
}
fn teach_hash(&self, path: &str, meta: NodeMeta) {
{
let mut learned = self.shared.learned.lock().unwrap();
if learned.len() >= 65536 {
learned.clear();
}
learned.insert(path.to_string(), meta.clone());
}
let _ = self.shared.tx.send(RootMsg::HashLearned {
path: path.to_string(),
meta,
});
}
fn rekey_move(&mut self, from: &str, to: &str) {
let moved: Vec<(String, u128)> = self
.held
.iter()
.filter(|(path, _)| is_under(path, from))
.map(|(path, &hash)| (path.clone(), hash))
.collect();
for (path, _) in &moved {
self.held.remove(path);
}
for (path, hash) in moved {
self.held.insert(rebase_subtree_path(&path, from, to), hash);
}
for path in subtree_keys(&self.retry, from) {
if let Some(entry) = self.retry.remove(&path) {
self.retry
.insert(rebase_subtree_path(&path, from, to), entry);
}
}
}
fn send_update(
&mut self,
records: Vec<u8>,
reset_pending: &mut bool,
sync: bool,
) -> Result<(), Exit> {
self.wait_for_credit()?;
let mut flags = 0u8;
if *reset_pending {
flags |= FS_UPDATE_RESET;
*reset_pending = false;
}
if sync {
flags |= FS_UPDATE_SYNC;
}
let update_id = self.next_update_id;
self.next_update_id = self.next_update_id.wrapping_add(1);
self.highest_sent = update_id;
let msg = msg_fs_update(self.sync_id, update_id, flags, &records);
self.unacked.push_back((update_id, msg.len()));
self.unacked_bytes += msg.len();
if !(self.outbox)(msg) {
return Err(Exit::ClientGone);
}
Ok(())
}
fn wait_for_credit(&mut self) -> Result<(), Exit> {
while self.unacked_bytes >= self.opts.window_bytes {
match self.rx.recv() {
Ok(SyncMsg::Cmd(Command::Ack(id))) => self.handle_ack(id)?,
Ok(SyncMsg::Cmd(Command::Fetch { nonce, path })) => {
if !self.handle_fetch(nonce, &path) {
return Err(Exit::ClientGone);
}
}
Ok(SyncMsg::Cmd(Command::Write(w))) => {
if !self.handle_write(w) {
return Err(Exit::ClientGone);
}
}
Ok(SyncMsg::Cmd(Command::Op(o))) => {
if !self.handle_op(o) {
return Err(Exit::ClientGone);
}
}
Ok(SyncMsg::Cmd(Command::Stop)) => return Err(Exit::Stopped),
Ok(SyncMsg::Root(update)) => self.handle_root(update)?,
Err(_) => return Err(Exit::ClientGone),
}
}
Ok(())
}
fn handle_fetch(&mut self, nonce: u16, wire_path: &str) -> bool {
if self.single {
let msg = if wire_path.is_empty() {
let root = self.root.clone();
self.fetch_confined(nonce, &root)
} else {
msg_fs_file(nonce, FS_FILE_NOT_FOUND, &[])
};
return (self.outbox)(msg);
}
let msg = match confine_target(&self.root, wire_path) {
Err(ConfineError::Invalid) => msg_fs_file(nonce, FS_FILE_NOT_FOUND, &[]),
Err(ConfineError::Io(e)) if e.kind() == io::ErrorKind::NotFound => {
msg_fs_file(nonce, FS_FILE_NOT_FOUND, &[])
}
Err(ConfineError::Io(_)) => msg_fs_file(nonce, FS_FILE_UNREADABLE, &[]),
Err(ConfineError::Escapes) => msg_fs_file(nonce, blit_remote::fs::FS_FILE_OTHER, &[]),
Ok(abs) => self.fetch_confined(nonce, &abs),
};
(self.outbox)(msg)
}
fn fetch_confined(&self, nonce: u16, abs: &Path) -> Vec<u8> {
let md = match fs::symlink_metadata(abs) {
Ok(md) => md,
Err(e) if e.kind() == io::ErrorKind::NotFound => {
return msg_fs_file(nonce, FS_FILE_NOT_FOUND, &[]);
}
Err(_) => return msg_fs_file(nonce, FS_FILE_UNREADABLE, &[]),
};
let ft = md.file_type();
if !ft.is_file() && !ft.is_symlink() {
return msg_fs_file(nonce, blit_remote::fs::FS_FILE_OTHER, &[]);
}
if ft.is_file() && md.len() > blit_remote::fs::FS_MAX_DECOMPRESSED as u64 {
return msg_fs_file(nonce, blit_remote::fs::FS_FILE_OTHER, &[]);
}
match read_verified(abs) {
ReadOutcome::Stable(data) => msg_fs_file(nonce, FS_FILE_OK, &data),
ReadOutcome::Unstable => msg_fs_file(nonce, FS_FILE_UNREADABLE, &[]),
ReadOutcome::Unreadable => {
if abs.exists() {
msg_fs_file(nonce, FS_FILE_UNREADABLE, &[])
} else {
msg_fs_file(nonce, FS_FILE_NOT_FOUND, &[])
}
}
}
}
fn resolve_target(&self, wire: &str, policy: SymlinkPolicy) -> Result<PathBuf, u8> {
if !self.single {
return resolve_write_target(&self.root, wire, policy);
}
if !wire.is_empty() {
return Err(FS_DONE_INVALID);
}
match fs::symlink_metadata(&self.root) {
Ok(md) if md.file_type().is_symlink() => match policy {
SymlinkPolicy::Refuse => Err(FS_DONE_PERMISSION),
SymlinkPolicy::Operate => Ok(self.root.clone()),
SymlinkPolicy::Follow => {
let resolved = fs::canonicalize(&self.root).map_err(|e| write_io_status(&e))?;
if resolved == self.root {
Ok(resolved)
} else {
Err(FS_DONE_PERMISSION)
}
}
},
_ => Ok(self.root.clone()),
}
}
fn handle_write(&mut self, w: WriteReq) -> bool {
let (status, hash, mtime_ns) = self.exec_write(&w);
(self.outbox)(msg_fs_done(w.nonce, status, hash, mtime_ns))
}
fn exec_write(&mut self, w: &WriteReq) -> (u8, u128, u64) {
use blit_remote::fs::{FS_WRITE_CONTENT_DELTA, FS_WRITE_CONTENT_FULL, apply_fs_delta};
let is_delta = w.content_kind == FS_WRITE_CONTENT_DELTA;
if !is_delta && w.content_kind != 0 && w.content_kind != FS_WRITE_CONTENT_FULL {
return (FS_DONE_INVALID, 0, 0);
}
let no_cas = w.flags & FS_WRITE_NO_CAS != 0;
if is_delta && (no_cas || w.base == 0) {
return (FS_DONE_INVALID, 0, 0);
}
if w.content.len() as u64 > fs_write_max() {
return (FS_DONE_TOO_LARGE, 0, 0);
}
if !self.single
&& w.flags & FS_WRITE_MKPARENTS != 0
&& let Some(parent) = resolve_wire_path(&self.root, &w.path)
.and_then(|a| a.parent().map(Path::to_path_buf))
&& let Err(status) = create_parents_confined(&self.root, &parent)
{
return (status, 0, 0);
}
let policy = if w.flags & FS_WRITE_FOLLOW_SYMLINK != 0 {
SymlinkPolicy::Follow
} else {
SymlinkPolicy::Refuse
};
let target = match self.resolve_target(&w.path, policy) {
Ok(t) => t,
Err(status) => return (status, 0, 0),
};
let durable = w.flags & FS_WRITE_DURABLE != 0;
let lock = path_write_lock(&target);
let _guard = lock.lock().unwrap();
if fs::symlink_metadata(&target)
.map(|m| m.is_dir())
.unwrap_or(false)
{
return (FS_DONE_WRONG_TYPE, 0, 0);
}
let create_exclusive_mode = !no_cas && w.base == 0;
let applied: Option<Vec<u8>> = if is_delta {
match fs::symlink_metadata(&target) {
Ok(md) if md.len() > fs_write_max() => return (FS_DONE_TOO_LARGE, 0, 0),
Err(e) if e.kind() == io::ErrorKind::NotFound => {
return (FS_DONE_CONFLICT, 0, 0);
}
_ => {}
}
let base = match read_verified_meta(&target) {
ReadMetaOutcome::Stable(data, _) => data,
ReadMetaOutcome::Unstable => return (FS_DONE_OTHER, 0, 0),
ReadMetaOutcome::Unreadable => {
return if target.exists() {
(FS_DONE_OTHER, 0, 0)
} else {
(FS_DONE_CONFLICT, 0, 0)
};
}
};
let cur = blake3_128(&base);
if cur != w.base {
return (FS_DONE_CONFLICT, cur, 0);
}
let Some(applied) = apply_fs_delta(&base, &w.content) else {
return (FS_DONE_INVALID, 0, 0);
};
if applied.len() as u64 > fs_write_max() {
return (FS_DONE_TOO_LARGE, 0, 0);
}
Some(applied)
} else {
if !no_cas {
if w.base == 0 {
if target.exists() {
return (FS_DONE_CONFLICT, current_hash(&target), 0);
}
} else {
let cur = current_hash(&target);
if cur != w.base {
return (FS_DONE_CONFLICT, cur, 0);
}
}
}
None
};
let content: &[u8] = applied.as_deref().unwrap_or(&w.content);
let hash = blake3_128(content);
if create_exclusive_mode {
match create_exclusive(&target, content, w.mode, durable) {
Ok(()) => {}
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {
return (FS_DONE_CONFLICT, current_hash(&target), 0);
}
Err(e) => return (write_io_status(&e), 0, 0),
}
} else if let Err(e) = write_atomic(&target, content, w.mode, durable) {
return (write_io_status(&e), 0, 0);
}
let mtime_ns = stat_meta(&target).map(|m| m.mtime_ns).unwrap_or(0);
let echo_wire = wire_key_for(&self.root, &target).unwrap_or_else(|| w.path.clone());
self.prime_echo(&echo_wire, &target, hash, content, mtime_ns);
(FS_DONE_OK, hash, mtime_ns)
}
fn handle_op(&mut self, o: OpReq) -> bool {
let (status, hash, mtime_ns) = self.exec_op(&o);
(self.outbox)(msg_fs_done(o.nonce, status, hash, mtime_ns))
}
fn exec_op(&mut self, o: &OpReq) -> (u8, u128, u64) {
match o.op {
FS_OP_MKDIR => {
if !self.single
&& o.flags & FS_OP_MKPARENTS != 0
&& let Some(parent) = resolve_wire_path(&self.root, &o.a)
.and_then(|a| a.parent().map(Path::to_path_buf))
&& let Err(status) = create_parents_confined(&self.root, &parent)
{
return (status, 0, 0);
}
let target = match self.resolve_target(&o.a, SymlinkPolicy::Operate) {
Ok(t) => t,
Err(status) => return (status, 0, 0),
};
let lock = path_write_lock(&target);
let _guard = lock.lock().unwrap();
let mut builder = fs::DirBuilder::new();
#[cfg(unix)]
if o.mode != 0 {
use std::os::unix::fs::DirBuilderExt;
builder.mode(o.mode);
}
match builder.create(&target) {
Ok(()) => {}
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {
if !target.is_dir() {
return (FS_DONE_CONFLICT, 0, 0);
}
}
Err(e) => return (write_io_status(&e), 0, 0),
}
let mtime_ns = stat_meta(&target).map(|m| m.mtime_ns).unwrap_or(0);
self.hint_change(&target);
(FS_DONE_OK, 0, mtime_ns)
}
FS_OP_REMOVE => {
let target = match self.resolve_target(&o.a, SymlinkPolicy::Operate) {
Ok(t) => t,
Err(status) => return (status, 0, 0),
};
let lock = path_write_lock(&target);
let _guard = lock.lock().unwrap();
let md = match fs::symlink_metadata(&target) {
Ok(m) => m,
Err(e) if e.kind() == io::ErrorKind::NotFound => {
return (FS_DONE_NOT_FOUND, 0, 0);
}
Err(e) => return (write_io_status(&e), 0, 0),
};
if o.flags & FS_OP_NO_CAS == 0 && o.base != 0 {
let cur = current_hash(&target);
if cur != o.base {
return (FS_DONE_CONFLICT, cur, 0);
}
}
let res = if md.file_type().is_dir() {
fs::remove_dir_all(&target)
} else {
fs::remove_file(&target)
};
if let Err(e) = res {
return (write_io_status(&e), 0, 0);
}
self.hint_change(&target);
(FS_DONE_OK, 0, 0)
}
FS_OP_RENAME => {
if !self.single
&& o.flags & FS_OP_MKPARENTS != 0
&& let Some(parent) = resolve_wire_path(&self.root, &o.b)
.and_then(|a| a.parent().map(Path::to_path_buf))
&& let Err(status) = create_parents_confined(&self.root, &parent)
{
return (status, 0, 0);
}
let from = match self.resolve_target(&o.a, SymlinkPolicy::Operate) {
Ok(t) => t,
Err(status) => return (status, 0, 0),
};
let lock = path_write_lock(&from);
let _guard = lock.lock().unwrap();
if fs::symlink_metadata(&from).is_err() {
return (FS_DONE_NOT_FOUND, 0, 0);
}
let to = match self.resolve_target(&o.b, SymlinkPolicy::Operate) {
Ok(t) => t,
Err(status) => return (status, 0, 0),
};
if let Err(e) = fs::rename(&from, &to) {
return (write_io_status(&e), 0, 0);
}
self.hint_change(&from);
self.hint_change(&to);
(FS_DONE_OK, 0, 0)
}
FS_OP_SYMLINK | FS_OP_HARDLINK => self.exec_link(o),
_ => (FS_DONE_INVALID, 0, 0),
}
}
fn exec_link(&mut self, o: &OpReq) -> (u8, u128, u64) {
if !self.single
&& o.flags & FS_OP_MKPARENTS != 0
&& let Some(parent) =
resolve_wire_path(&self.root, &o.b).and_then(|b| b.parent().map(Path::to_path_buf))
&& let Err(status) = create_parents_confined(&self.root, &parent)
{
return (status, 0, 0);
}
let src = if o.op == FS_OP_HARDLINK {
let src = match self.resolve_target(&o.a, SymlinkPolicy::Operate) {
Ok(t) => t,
Err(status) => return (status, 0, 0),
};
match fs::symlink_metadata(&src) {
Ok(md) if md.file_type().is_file() => {}
Ok(_) => return (FS_DONE_WRONG_TYPE, 0, 0),
Err(e) if e.kind() == io::ErrorKind::NotFound => {
return (FS_DONE_NOT_FOUND, 0, 0);
}
Err(e) => return (write_io_status(&e), 0, 0),
}
Some(src)
} else {
if o.a.is_empty() {
return (FS_DONE_INVALID, 0, 0);
}
None
};
let link = match self.resolve_target(&o.b, SymlinkPolicy::Operate) {
Ok(t) => t,
Err(status) => return (status, 0, 0),
};
let lock = path_write_lock(&link);
let _guard = lock.lock().unwrap();
if fs::symlink_metadata(&link)
.map(|m| m.is_dir())
.unwrap_or(false)
{
return (FS_DONE_WRONG_TYPE, 0, 0);
}
let no_cas = o.flags & FS_OP_NO_CAS != 0;
let create_exclusive_mode = !no_cas && o.base == 0;
if !no_cas {
if o.base == 0 {
if fs::symlink_metadata(&link).is_ok() {
return (FS_DONE_CONFLICT, current_hash(&link), 0);
}
} else {
let cur = current_hash(&link);
if cur != o.base {
return (FS_DONE_CONFLICT, cur, 0);
}
}
}
let create = |at: &Path| -> io::Result<()> {
match &src {
Some(src) => fs::hard_link(src, at),
None => symlink_at(&o.a, at),
}
};
if create_exclusive_mode {
match create(&link) {
Ok(()) => {}
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {
return (FS_DONE_CONFLICT, current_hash(&link), 0);
}
Err(e) => return (write_io_status(&e), 0, 0),
}
} else {
let tmp = temp_sibling(&link);
if let Err(e) = create(&tmp) {
return (write_io_status(&e), 0, 0);
}
if let Err(e) = fs::rename(&tmp, &link) {
let _ = fs::remove_file(&tmp);
return (write_io_status(&e), 0, 0);
}
}
let mtime_ns = stat_meta(&link).map(|m| m.mtime_ns).unwrap_or(0);
let echo_wire = wire_key_for(&self.root, &link).unwrap_or_else(|| o.b.clone());
match &src {
None => {
let hash = blake3_128(o.a.as_bytes());
self.prime_echo(&echo_wire, &link, hash, o.a.as_bytes(), mtime_ns);
(FS_DONE_OK, hash, mtime_ns)
}
Some(src) => {
let small = fs::symlink_metadata(src)
.map(|m| m.len() <= fs_write_max())
.unwrap_or(false);
match if small {
read_verified(&link)
} else {
ReadOutcome::Unstable
} {
ReadOutcome::Stable(data) => {
let hash = blake3_128(&data);
self.prime_echo(&echo_wire, &link, hash, &data, mtime_ns);
(FS_DONE_OK, hash, mtime_ns)
}
_ => {
self.hint_change(&link);
(FS_DONE_OK, 0, mtime_ns)
}
}
}
}
}
fn prime_echo(&mut self, wire: &str, abs: &Path, hash: u128, bytes: &[u8], mtime_ns: u64) {
blob_store()
.lock()
.unwrap()
.put(hash, Arc::new(bytes.to_vec()));
self.held.insert(wire.to_string(), hash);
if !racily_clean(mtime_ns)
&& let Ok(mut meta) = stat_meta(abs)
{
meta.hash = hash;
self.teach_hash(wire, meta);
}
self.hint_change(abs);
}
fn hint_change(&self, abs: &Path) {
let _ = self
.shared
.tx
.send(RootMsg::Hint(Hint::Dirty(abs.to_path_buf())));
if let Some(parent) = abs.parent() {
let _ = self
.shared
.tx
.send(RootMsg::Hint(Hint::Dirty(parent.to_path_buf())));
}
}
}
const FS_CLOSED_BACKEND_FAILED_COMPAT: u8 = blit_remote::fs::FS_CLOSED_BACKEND_FAILED;
const FS_CLOSED_PERMISSION_LOST_COMPAT: u8 = blit_remote::fs::FS_CLOSED_PERMISSION_LOST;
const RESOURCE_LIMIT_ERRNO: i32 = libc_enfile();
const fn libc_enfile() -> i32 {
23 }
#[cfg(test)]
mod tests {
use super::*;
use blit_remote::fs::FsMirror;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
static TEST_DIR_SEQ: AtomicU64 = AtomicU64::new(0);
fn temp_dir() -> PathBuf {
let dir = std::env::temp_dir().join(format!(
"blit-fssync-test-{}-{}",
std::process::id(),
TEST_DIR_SEQ.fetch_add(1, Ordering::Relaxed)
));
fs::create_dir_all(&dir).unwrap();
dir
}
fn test_key(root: &Path) -> RootKey {
RootKey {
path: root.to_path_buf(),
recursive: true,
cross_filesystem: false,
ignores: IgnoreSpec::default(),
}
}
fn test_key_ignoring(root: &Path, ignores: IgnoreSpec) -> RootKey {
RootKey {
ignores,
..test_key(root)
}
}
#[test]
fn escape_roundtrip() {
assert_eq!(escape_bytes(b"plain.txt"), "plain.txt");
assert_eq!(escape_bytes(b"50%.txt"), "50%25.txt");
let bad = b"a\xFFb";
let escaped = escape_bytes(bad);
assert_eq!(escaped, "a%FFb");
assert_eq!(unescape_to_bytes(&escaped).unwrap(), bad.to_vec());
assert_eq!(unescape_to_bytes("50%25.txt").unwrap(), b"50%.txt".to_vec());
}
#[test]
fn wide_escape_roundtrip() {
let plain: Vec<u16> = "file.txt".encode_utf16().collect();
assert_eq!(escape_wide(&plain), "file.txt");
assert_eq!(unescape_to_wide("file.txt").unwrap(), plain);
let percent: Vec<u16> = "50%u.txt".encode_utf16().collect();
assert_eq!(escape_wide(&percent), "50%25u.txt");
assert_eq!(unescape_to_wide("50%25u.txt").unwrap(), percent);
let clef: Vec<u16> = "𝄞.txt".encode_utf16().collect();
assert_eq!(escape_wide(&clef), "𝄞.txt");
assert_eq!(unescape_to_wide("𝄞.txt").unwrap(), clef);
let bad = [0xD800u16, 0x0041, 0xDFFF];
let escaped = escape_wide(&bad);
assert_eq!(escaped, "%uD800A%uDFFF");
assert_eq!(unescape_to_wide(&escaped).unwrap(), bad.to_vec());
assert!(unescape_to_wide("%u12").is_none());
assert!(unescape_to_wide("%uZZZZ").is_none());
}
#[test]
fn wire_path_traversal_rejected() {
let root = Path::new("/tmp/root");
assert!(resolve_wire_path(root, "a/../b").is_none());
assert!(resolve_wire_path(root, "..").is_none());
assert!(resolve_wire_path(root, "a//b").is_none());
assert_eq!(resolve_wire_path(root, ""), Some(root.to_path_buf()));
assert_eq!(
resolve_wire_path(root, "a/b"),
Some(root.join("a").join("b"))
);
}
#[test]
fn encoded_traversal_rejected() {
let root = Path::new("/tmp/root");
assert!(resolve_wire_path(root, "%2E%2E").is_none());
assert!(resolve_wire_path(root, "%2e%2e/etc/passwd").is_none());
assert!(resolve_wire_path(root, "%2E").is_none());
assert!(resolve_wire_path(root, "a%2F..%2Fb").is_none());
assert!(resolve_wire_path(root, "a%2Fb").is_none());
assert_eq!(resolve_wire_path(root, "%2525"), Some(root.join("%25")));
}
fn meta(node_type: u8, size: u64, mtime: u64, ino: u64) -> NodeMeta {
NodeMeta {
node_type,
size,
mtime_ns: mtime,
mode: 0o644,
hash: 0,
dev_ino: (1, ino),
link_dir: false,
filtered: false,
}
}
#[test]
fn diff_reports_a_flag_flip_under_an_unchanged_stat() {
for (label, flip) in [
(
"filtered",
(|m: &mut NodeMeta| m.filtered = true) as fn(&mut NodeMeta),
),
("link_dir", |m: &mut NodeMeta| m.link_dir = true),
] {
let mut prev = Index::new();
prev.insert("".into(), meta(FS_ENTRY_DIR, 0, 0, 1));
prev.insert("d".into(), meta(FS_ENTRY_DIR, 0, 0, 2));
let mut curr = prev.clone();
flip(curr.get_mut("d").unwrap());
let changed = std::collections::BTreeSet::from(["d".to_string()]);
for (how, ops) in [
("diff", diff(&prev, &curr)),
("diff_changed", diff_changed(&prev, &curr, &changed)),
] {
let [
DiffOp::Upsert {
path,
content_changed,
},
] = &ops[..]
else {
panic!("{label} via {how}: expected one Upsert, got {ops:?}");
};
assert_eq!(path, "d", "{label} via {how}");
assert!(!content_changed, "{label} via {how} asked for content");
}
}
}
#[test]
fn diff_detects_directory_move() {
let mut prev = Index::new();
prev.insert("".into(), meta(FS_ENTRY_DIR, 0, 0, 1));
prev.insert("d".into(), meta(FS_ENTRY_DIR, 0, 0, 2));
prev.insert("d/f".into(), meta(FS_ENTRY_FILE, 5, 10, 3));
let mut curr = Index::new();
curr.insert("".into(), meta(FS_ENTRY_DIR, 0, 0, 1));
curr.insert("e".into(), meta(FS_ENTRY_DIR, 0, 0, 2));
curr.insert("e/f".into(), meta(FS_ENTRY_FILE, 5, 10, 3));
let ops = diff(&prev, &curr);
assert_eq!(
ops,
vec![DiffOp::Move {
from: "d".into(),
to: "e".into()
}]
);
}
#[test]
fn diff_move_with_same_window_child_changes() {
let mut prev = Index::new();
prev.insert("".into(), meta(FS_ENTRY_DIR, 0, 0, 1));
prev.insert("d".into(), meta(FS_ENTRY_DIR, 0, 50, 2));
prev.insert("d/modified".into(), meta(FS_ENTRY_FILE, 5, 10, 3));
prev.insert("d/deleted".into(), meta(FS_ENTRY_FILE, 5, 10, 4));
let mut curr = Index::new();
curr.insert("".into(), meta(FS_ENTRY_DIR, 0, 0, 1));
curr.insert("e".into(), meta(FS_ENTRY_DIR, 0, 50, 2));
curr.insert("e/modified".into(), meta(FS_ENTRY_FILE, 999, 777, 3));
curr.insert("e/created".into(), meta(FS_ENTRY_FILE, 1, 900, 9));
let ops = diff(&prev, &curr);
assert!(ops.contains(&DiffOp::Move {
from: "d".into(),
to: "e".into()
}));
assert!(
ops.contains(&DiffOp::Upsert {
path: "e/modified".into(),
content_changed: true
}),
"modified child swallowed: {ops:?}"
);
assert!(
ops.contains(&DiffOp::Upsert {
path: "e/created".into(),
content_changed: true
}),
"created child swallowed: {ops:?}"
);
assert!(
ops.contains(&DiffOp::Delete {
path: "e/deleted".into()
}),
"deleted child swallowed: {ops:?}"
);
}
#[cfg(unix)]
fn drive_engine(root: &Path) -> (Arc<Mutex<Vec<Vec<u8>>>>, SyncHandle, HintSender) {
drive_engine_keyed(test_key(root))
}
fn drive_engine_keyed(key: RootKey) -> (Arc<Mutex<Vec<Vec<u8>>>>, SyncHandle, HintSender) {
let shared = open_root_unwatched(key);
let hint_tx = shared.hint_sender();
let sent: Arc<Mutex<Vec<Vec<u8>>>> = Default::default();
let sent2 = sent.clone();
let opts = SyncOptions {
content: true,
latency: Duration::from_millis(5),
..Default::default()
};
let handle = start_sync(
&shared,
1,
opts,
Box::new(move |msg| {
sent2.lock().unwrap().push(msg);
true
}),
);
(sent, handle, hint_tx)
}
fn await_done(
handle: &SyncHandle,
sent: &Arc<Mutex<Vec<Vec<u8>>>>,
nonce: u16,
cmd: Command,
) -> (u8, u128, u64) {
handle.command(cmd);
let deadline = Instant::now() + Duration::from_secs(5);
loop {
for msg in sent.lock().unwrap().iter() {
if let Some((n, s, h, m)) = blit_remote::fs::parse_fs_done(msg)
&& n == nonce
{
return (s, h, m);
}
}
assert!(Instant::now() < deadline, "no FS_DONE for nonce {nonce}");
std::thread::sleep(Duration::from_millis(2));
}
}
fn write_req(nonce: u16, path: &str, base: u128, flags: u8, content: &[u8]) -> Command {
Command::Write(WriteReq {
nonce,
path: path.into(),
base,
mode: 0,
flags,
content_kind: 1,
content: content.to_vec(),
inflight: None,
})
}
fn drive_single_engine(file: &Path) -> (Arc<Mutex<Vec<Vec<u8>>>>, SyncHandle, HintSender) {
let shared = open_single_root_unwatched(file.to_path_buf());
assert!(shared.is_single());
let hint_tx = shared.hint_sender();
let sent: Arc<Mutex<Vec<Vec<u8>>>> = Default::default();
let sent2 = sent.clone();
let opts = SyncOptions {
content: true,
recursive: false,
latency: Duration::from_millis(5),
..Default::default()
};
let handle = start_sync(
&shared,
1,
opts,
Box::new(move |msg| {
sent2.lock().unwrap().push(msg);
true
}),
);
(sent, handle, hint_tx)
}
fn count_updates(sent: &Arc<Mutex<Vec<Vec<u8>>>>) -> usize {
sent.lock()
.unwrap()
.iter()
.filter(|m| m[0] == blit_remote::fs::S2C_FS_UPDATE)
.count()
}
fn count_closed(sent: &Arc<Mutex<Vec<Vec<u8>>>>) -> usize {
sent.lock()
.unwrap()
.iter()
.filter(|m| m[0] == blit_remote::fs::S2C_FS_CLOSED)
.count()
}
fn pump_mirror(
sent: &Arc<Mutex<Vec<Vec<u8>>>>,
handle: &SyncHandle,
mirror: &mut FsMirror,
seen: &mut usize,
) {
let msgs = sent.lock().unwrap().clone();
for msg in &msgs[*seen..] {
if msg[0] == blit_remote::fs::S2C_FS_UPDATE {
let id = mirror.apply_update(msg).expect("valid update");
handle.command(Command::Ack(id));
}
}
*seen = msgs.len();
}
fn pump_until(
sent: &Arc<Mutex<Vec<Vec<u8>>>>,
handle: &SyncHandle,
mirror: &mut FsMirror,
seen: &mut usize,
what: &str,
pred: impl Fn(&FsMirror) -> bool,
) {
pump_until_nudging(sent, handle, mirror, seen, what, || {}, pred)
}
fn pump_until_nudging(
sent: &Arc<Mutex<Vec<Vec<u8>>>>,
handle: &SyncHandle,
mirror: &mut FsMirror,
seen: &mut usize,
what: &str,
nudge: impl Fn(),
pred: impl Fn(&FsMirror) -> bool,
) {
let deadline = Instant::now() + Duration::from_secs(30);
loop {
pump_mirror(sent, handle, mirror, seen);
if pred(mirror) {
return;
}
assert!(
Instant::now() < deadline,
"timed out waiting for {what}; live = {:?}",
mirror.live.keys().collect::<Vec<_>>()
);
nudge();
std::thread::sleep(Duration::from_millis(2));
}
}
#[test]
fn single_sync_lifecycle() {
let dir = temp_dir().canonicalize().unwrap();
let file = dir.join("note.txt");
let sibling = dir.join("sibling.txt");
fs::write(&file, b"v1").unwrap();
fs::write(&sibling, b"noise").unwrap();
let shared = open_single_root_unwatched(file.clone());
assert!(Arc::ptr_eq(
&shared,
&open_single_root_unwatched(file.clone())
));
let dir_root = open_root_unwatched(test_key(&dir));
assert!(!Arc::ptr_eq(&shared, &dir_root));
drop(dir_root);
drop(shared);
let (sent, handle, hint) = drive_single_engine(&file);
let mut mirror = FsMirror::new();
let mut seen = 0usize;
pump_until(&sent, &handle, &mut mirror, &mut seen, "initial ''", |m| {
m.live
.get("")
.is_some_and(|n| n.content.as_deref() == Some(&b"v1"[..]))
});
assert_eq!(mirror.live.len(), 1, "mirror holds exactly the root");
let node = &mirror.live[""];
assert_eq!(node.entry_flags & FS_ENTRY_TYPE_MASK, FS_ENTRY_FILE);
assert_eq!(node.hash, blake3_128(b"v1"));
let quiet = count_updates(&sent);
fs::write(&sibling, b"more noise").unwrap();
fs::write(dir.join("new-sibling.txt"), b"x").unwrap();
hint.send(Hint::Dirty(sibling.clone()));
hint.send(Hint::Dirty(dir.join("new-sibling.txt")));
std::thread::sleep(Duration::from_millis(120));
pump_mirror(&sent, &handle, &mut mirror, &mut seen);
assert_eq!(
count_updates(&sent),
quiet,
"sibling churn woke the single sync"
);
assert_eq!(mirror.live[""].content.as_deref(), Some(&b"v1"[..]));
fs::write(&file, b"v2").unwrap();
hint.send(Hint::Dirty(file.clone()));
pump_until(&sent, &handle, &mut mirror, &mut seen, "v2", |m| {
m.live
.get("")
.is_some_and(|n| n.content.as_deref() == Some(&b"v2"[..]))
});
fs::write(&file, b"v3").unwrap();
hint.send(Hint::Dirty(dir.clone()));
pump_until(&sent, &handle, &mut mirror, &mut seen, "v3", |m| {
m.live
.get("")
.is_some_and(|n| n.content.as_deref() == Some(&b"v3"[..]))
});
fs::remove_file(&file).unwrap();
hint.send(Hint::Dirty(file.clone()));
pump_until(&sent, &handle, &mut mirror, &mut seen, "delete", |m| {
m.live.is_empty()
});
assert_eq!(count_closed(&sent), 0, "delete must not close the sync");
fs::write(&file, b"v4").unwrap();
hint.send(Hint::Dirty(file.clone()));
pump_until(&sent, &handle, &mut mirror, &mut seen, "recreate", |m| {
m.live
.get("")
.is_some_and(|n| n.content.as_deref() == Some(&b"v4"[..]))
});
let away = dir.join("renamed.txt");
fs::rename(&file, &away).unwrap();
hint.send(Hint::Dirty(file.clone()));
hint.send(Hint::Dirty(away.clone()));
pump_until(&sent, &handle, &mut mirror, &mut seen, "rename away", |m| {
m.live.is_empty()
});
fs::rename(&away, &file).unwrap();
hint.send(Hint::Dirty(file.clone()));
hint.send(Hint::Dirty(away.clone()));
pump_until(&sent, &handle, &mut mirror, &mut seen, "rename back", |m| {
m.live
.get("")
.is_some_and(|n| n.content.as_deref() == Some(&b"v4"[..]))
});
assert_eq!(count_closed(&sent), 0);
handle.command(Command::Stop);
let deadline = Instant::now() + Duration::from_secs(5);
while count_closed(&sent) == 0 {
assert!(Instant::now() < deadline, "no FS_CLOSED after Stop");
std::thread::sleep(Duration::from_millis(2));
}
let closed = sent
.lock()
.unwrap()
.iter()
.find(|m| m[0] == blit_remote::fs::S2C_FS_CLOSED)
.unwrap()
.clone();
assert_eq!(closed[3], FS_CLOSED_CLIENT_REQUEST);
let _ = fs::remove_dir_all(&dir);
}
#[cfg(target_os = "linux")]
#[test]
fn cross_device_symlink_is_not_descended_on_reconcile() {
use std::os::unix::fs::MetadataExt;
let dir = temp_dir().canonicalize().unwrap();
let Ok(shm) = std::path::Path::new("/dev/shm").canonicalize() else {
return;
};
let foreign = shm.join(format!("blit-xdev-{}", std::process::id()));
if fs::create_dir_all(foreign.join("inner")).is_err() {
return;
}
let (Ok(a), Ok(b)) = (fs::metadata(&dir), fs::metadata(&foreign)) else {
let _ = fs::remove_dir_all(&foreign);
return;
};
if a.dev() == b.dev() {
let _ = fs::remove_dir_all(&foreign);
return;
}
fs::write(foreign.join("inner/secret.txt"), b"elsewhere").unwrap();
fs::write(dir.join("local.txt"), b"here").unwrap();
let (sent, handle, hint) = drive_engine(&dir);
let mut mirror = FsMirror::new();
let mut seen = 0usize;
pump_until(&sent, &handle, &mut mirror, &mut seen, "initial", |m| {
m.live.contains_key("local.txt")
});
std::os::unix::fs::symlink(&foreign, dir.join("far")).unwrap();
hint.send(Hint::Dirty(dir.join("far")));
pump_until(&sent, &handle, &mut mirror, &mut seen, "link entry", |m| {
m.live.contains_key("far")
});
let leaked: Vec<String> = mirror
.live
.keys()
.filter(|k| k.starts_with("far/"))
.cloned()
.collect();
handle.command(Command::Stop);
let _ = fs::remove_dir_all(&foreign);
assert!(
leaked.is_empty(),
"cross_filesystem is off: a symlink to another device must not be \
descended, found {leaked:?}"
);
}
#[cfg(target_os = "linux")]
#[test]
fn per_directory_watching_still_delivers_every_change() {
let root = temp_dir().canonicalize().unwrap();
fs::create_dir_all(root.join("src/deep")).unwrap();
fs::create_dir_all(root.join("node_modules/pkg")).unwrap();
fs::write(root.join(".gitignore"), "node_modules/\n").unwrap();
fs::write(root.join("src/deep/seed.txt"), b"seed").unwrap();
let key = test_key_ignoring(
&root,
IgnoreSpec {
gitignore: true,
dot_ignore: true,
exclude_git: true,
patterns: Vec::new(),
},
);
let shared = open_root(key).expect("arm native watch");
assert!(
shared
._backend
.lock()
.unwrap()
.as_ref()
.is_some_and(|b| b.watches.is_per_dir()),
"a filtered root on Linux arms per directory"
);
let sent: Arc<Mutex<Vec<Vec<u8>>>> = Default::default();
let sent2 = sent.clone();
let handle = start_sync(
&shared,
9,
SyncOptions {
content: true,
latency: Duration::from_millis(5),
..Default::default()
},
Box::new(move |msg| {
sent2.lock().unwrap().push(msg);
true
}),
);
let mut mirror = FsMirror::new();
let mut seen = 0usize;
pump_until(&sent, &handle, &mut mirror, &mut seen, "initial", |m| {
m.live.contains_key("src/deep/seed.txt")
});
fs::write(root.join("src/deep/seed.txt"), b"changed").unwrap();
pump_until(&sent, &handle, &mut mirror, &mut seen, "deep write", |m| {
m.live
.get("src/deep/seed.txt")
.is_some_and(|n| n.content.as_deref() == Some(&b"changed"[..]))
});
fs::create_dir(root.join("src/fresh")).unwrap();
fs::write(root.join("src/fresh/a.txt"), b"a").unwrap();
pump_until(&sent, &handle, &mut mirror, &mut seen, "fresh dir", |m| {
m.live.contains_key("src/fresh/a.txt")
});
fs::write(root.join("src/fresh/b.txt"), b"b").unwrap();
pump_until(&sent, &handle, &mut mirror, &mut seen, "fresh child", |m| {
m.live.contains_key("src/fresh/b.txt")
});
fs::remove_dir_all(root.join("src/fresh")).unwrap();
pump_until(&sent, &handle, &mut mirror, &mut seen, "dir gone", |m| {
!m.live.contains_key("src/fresh")
});
fs::create_dir(root.join("src/fresh")).unwrap();
fs::write(root.join("src/fresh/c.txt"), b"c").unwrap();
pump_until(&sent, &handle, &mut mirror, &mut seen, "re-armed", |m| {
m.live.contains_key("src/fresh/c.txt")
});
fs::write(root.join("node_modules/pkg/index.js"), b"x").unwrap();
std::thread::sleep(Duration::from_millis(100));
pump_mirror(&sent, &handle, &mut mirror, &mut seen);
assert!(
!mirror.live.keys().any(|k| k.starts_with("node_modules")),
"live = {:?}",
mirror.live.keys().collect::<Vec<_>>()
);
handle.command(Command::Stop);
}
#[test]
fn excluded_paths_never_reach_the_client() {
let dir = temp_dir().canonicalize().unwrap();
fs::create_dir_all(dir.join(".git")).unwrap();
fs::write(dir.join(".git/config"), b"[core]").unwrap();
fs::create_dir_all(dir.join("node_modules/pkg")).unwrap();
fs::write(dir.join("node_modules/pkg/index.js"), b"x").unwrap();
fs::create_dir_all(dir.join("target/debug")).unwrap();
fs::write(dir.join("target/debug/bin"), b"x").unwrap();
fs::create_dir_all(dir.join("src")).unwrap();
fs::write(dir.join("src/a.rs"), b"fn main() {}").unwrap();
fs::write(dir.join(".gitignore"), "target/\nnode_modules/\n").unwrap();
let key = test_key_ignoring(
&dir,
IgnoreSpec {
gitignore: true,
dot_ignore: true,
exclude_git: true,
patterns: Vec::new(),
},
);
let (sent, handle, hint) = drive_engine_keyed(key);
let mut mirror = FsMirror::new();
let mut seen = 0usize;
pump_until(&sent, &handle, &mut mirror, &mut seen, "initial", |m| {
m.live.contains_key("src/a.rs")
});
assert_eq!(
mirror.live.keys().cloned().collect::<Vec<_>>(),
["", ".gitignore", "src", "src/a.rs"],
"the whole checkout, and nothing the exclusions cover"
);
let quiet = count_updates(&sent);
fs::write(dir.join("target/debug/fresh.bin"), b"y").unwrap();
fs::write(dir.join(".git/HEAD"), b"ref: refs/heads/main").unwrap();
hint.send(Hint::Dirty(dir.join("target/debug/fresh.bin")));
hint.send(Hint::Dirty(dir.join(".git/HEAD")));
std::thread::sleep(Duration::from_millis(50));
assert_eq!(count_updates(&sent), quiet, "excluded churn woke the sync");
fs::write(dir.join("src/b.rs"), b"pub fn b() {}").unwrap();
hint.send(Hint::Dirty(dir.join("src/b.rs")));
pump_until(&sent, &handle, &mut mirror, &mut seen, "src/b.rs", |m| {
m.live.contains_key("src/b.rs")
});
assert!(
!mirror
.live
.keys()
.any(|k| k.starts_with("target") || k.starts_with(".git/")),
"live = {:?}",
mirror.live.keys().collect::<Vec<_>>()
);
fs::write(dir.join(".gitignore"), "target/\nnode_modules/\nsrc/a.rs\n").unwrap();
hint.send(Hint::Dirty(dir.join(".gitignore")));
pump_until(&sent, &handle, &mut mirror, &mut seen, "a.rs gone", |m| {
!m.live.contains_key("src/a.rs")
});
assert!(mirror.live.contains_key("src/b.rs"), "only the rule's path");
fs::write(dir.join(".gitignore"), "target/\nnode_modules/\n").unwrap();
hint.send(Hint::Dirty(dir.join(".gitignore")));
pump_until(&sent, &handle, &mut mirror, &mut seen, "a.rs back", |m| {
m.live.contains_key("src/a.rs")
});
handle.command(Command::Stop);
}
#[cfg(target_os = "linux")]
#[test]
fn an_edit_to_an_ignore_file_above_the_root_reaches_the_client() {
let top = temp_dir().canonicalize().unwrap();
fs::create_dir_all(top.join(".git")).unwrap();
let root = top.join("crates");
fs::create_dir_all(&root).unwrap();
fs::write(top.join(".gitignore"), "*.bak\n").unwrap();
fs::write(root.join("a.rs"), b"x").unwrap();
fs::write(root.join("old.bak"), b"x").unwrap();
let key = test_key_ignoring(
&root,
IgnoreSpec {
gitignore: true,
..Default::default()
},
);
let shared = open_root(key).expect("arm native watch");
let sent: Arc<Mutex<Vec<Vec<u8>>>> = Default::default();
let sent2 = sent.clone();
let handle = start_sync(
&shared,
9,
SyncOptions {
latency: Duration::from_millis(5),
..Default::default()
},
Box::new(move |msg| {
sent2.lock().unwrap().push(msg);
true
}),
);
let mut mirror = FsMirror::new();
let mut seen = 0usize;
pump_until(&sent, &handle, &mut mirror, &mut seen, "initial", |m| {
m.live.contains_key("a.rs")
});
assert!(!mirror.live.contains_key("old.bak"), "inherited from above");
fs::write(top.join(".gitignore"), "*.tmp\n").unwrap();
pump_until(&sent, &handle, &mut mirror, &mut seen, "uncovered", |m| {
m.live.contains_key("old.bak")
});
fs::write(top.join(".gitignore"), "*.rs\n").unwrap();
pump_until(&sent, &handle, &mut mirror, &mut seen, "covered", |m| {
!m.live.contains_key("a.rs")
});
handle.command(Command::Stop);
}
#[test]
fn a_directory_pattern_excludes_a_symlinked_directory_and_its_subtree() {
let dir = temp_dir().canonicalize().unwrap();
fs::create_dir_all(dir.join("real/inner")).unwrap();
fs::write(dir.join("real/inner/heavy.bin"), b"x").unwrap();
fs::write(dir.join("keep.txt"), b"k").unwrap();
std::os::unix::fs::symlink(dir.join("real"), dir.join("build")).unwrap();
let key = test_key_ignoring(
&dir,
IgnoreSpec {
patterns: vec!["build/".into()],
..Default::default()
},
);
let (sent, handle, _hint) = drive_engine_keyed(key);
let mut mirror = FsMirror::new();
let mut seen = 0usize;
pump_until(&sent, &handle, &mut mirror, &mut seen, "initial", |m| {
m.live.contains_key("keep.txt")
});
assert!(
!mirror.live.keys().any(|k| k.starts_with("build")),
"the link and everything enumerated through it; live = {:?}",
mirror.live.keys().collect::<Vec<_>>()
);
assert!(mirror.live.contains_key("real/inner/heavy.bin"));
handle.command(Command::Stop);
}
#[test]
fn a_directory_reports_that_it_hid_children() {
let dir = temp_dir().canonicalize().unwrap();
fs::create_dir_all(dir.join("src")).unwrap();
fs::create_dir_all(dir.join("plain")).unwrap();
fs::write(dir.join("src/a.rs"), b"x").unwrap();
fs::write(dir.join("src/a.tmp"), b"x").unwrap();
fs::write(dir.join("plain/b.rs"), b"x").unwrap();
let key = test_key_ignoring(
&dir,
IgnoreSpec {
patterns: vec!["*.tmp".into()],
..Default::default()
},
);
let (sent, handle, hint) = drive_engine_keyed(key);
let mut mirror = FsMirror::new();
let mut seen = 0usize;
pump_until(&sent, &handle, &mut mirror, &mut seen, "initial", |m| {
m.live.contains_key("plain/b.rs")
});
let filtered = |m: &FsMirror, path: &str| {
m.live
.get(path)
.is_some_and(|n| n.entry_flags & FS_ENTRY_FILTERED != 0)
};
assert!(filtered(&mirror, "src"), "src hid a.tmp");
assert!(!filtered(&mirror, "plain"), "plain hid nothing");
assert!(!filtered(&mirror, ""), "nor did the root");
fs::write(dir.join("plain/c.tmp"), b"x").unwrap();
hint.send(Hint::Dirty(dir.join("plain/c.tmp")));
pump_until_nudging(
&sent,
&handle,
&mut mirror,
&mut seen,
"plain hides",
|| {
hint.send(Hint::Dirty(dir.join("plain/c.tmp")));
},
|m| {
m.live
.get("plain")
.is_some_and(|n| n.entry_flags & FS_ENTRY_FILTERED != 0)
},
);
fs::remove_file(dir.join("plain/c.tmp")).unwrap();
hint.send(Hint::Dirty(dir.join("plain")));
pump_until_nudging(
&sent,
&handle,
&mut mirror,
&mut seen,
"plain clears",
|| {
hint.send(Hint::Dirty(dir.join("plain")));
},
|m| {
m.live
.get("plain")
.is_some_and(|n| n.entry_flags & FS_ENTRY_FILTERED == 0)
},
);
assert!(filtered(&mirror, "src"), "src still hides a.tmp");
handle.command(Command::Stop);
}
#[test]
fn client_patterns_outrank_ignore_files_and_key_the_root() {
let dir = temp_dir().canonicalize().unwrap();
fs::write(dir.join(".gitignore"), "*.log\n").unwrap();
fs::write(dir.join("a.log"), b"x").unwrap();
fs::write(dir.join("keep.log"), b"x").unwrap();
fs::write(dir.join("notes.txt"), b"x").unwrap();
let spec = IgnoreSpec {
gitignore: true,
dot_ignore: true,
exclude_git: false,
patterns: IgnoreSpec::parse_patterns("!keep.log\nnotes.txt"),
};
let (sent, handle, _hint) = drive_engine_keyed(test_key_ignoring(&dir, spec.clone()));
let mut mirror = FsMirror::new();
let mut seen = 0usize;
pump_until(&sent, &handle, &mut mirror, &mut seen, "initial", |m| {
m.live.contains_key("keep.log")
});
assert_eq!(
mirror.live.keys().cloned().collect::<Vec<_>>(),
["", ".gitignore", "keep.log"]
);
let same = open_root_unwatched(test_key_ignoring(&dir, spec.clone()));
let again = open_root_unwatched(test_key_ignoring(&dir, spec));
assert!(Arc::ptr_eq(&same, &again), "one spec, one shared root");
let unfiltered = open_root_unwatched(test_key(&dir));
assert!(
!Arc::ptr_eq(&same, &unfiltered),
"an unfiltered sync indexes a different tree"
);
handle.command(Command::Stop);
}
#[test]
fn percent_in_root_survives_the_wire_round_trip() {
let dir = temp_dir().canonicalize().unwrap();
let literal = dir.join("50%.txt");
fs::write(&literal, b"x").unwrap();
let echoed = escape_path(&literal);
assert!(echoed.ends_with("50%25.txt"), "echo escapes the percent");
assert_eq!(
validate_root(&literal.to_string_lossy()).unwrap(),
literal.canonicalize().unwrap(),
"a raw path containing % still works"
);
assert_eq!(
validate_root(&echoed).unwrap(),
literal.canonicalize().unwrap(),
"the escaped echo resolves back to the same file"
);
let ambiguous = dir.join("50%25.txt");
fs::write(&ambiguous, b"y").unwrap();
assert_eq!(
validate_root(&echoed).unwrap(),
ambiguous.canonicalize().unwrap(),
"literal match takes precedence over the decoded one"
);
}
#[cfg(unix)]
#[test]
fn symlinked_directories_are_traversed_and_cycle_safe() {
let dir = temp_dir().canonicalize().unwrap();
fs::create_dir_all(dir.join("real/inner")).unwrap();
fs::write(dir.join("real/inner/deep.txt"), b"payload").unwrap();
fs::write(dir.join("real/top.txt"), b"top").unwrap();
std::os::unix::fs::symlink(dir.join("real"), dir.join("link")).unwrap();
std::os::unix::fs::symlink(dir.join("real"), dir.join("real/loop")).unwrap();
std::os::unix::fs::symlink(dir.join("real/top.txt"), dir.join("tolink")).unwrap();
std::os::unix::fs::symlink(dir.join("nope"), dir.join("dangling")).unwrap();
let (sent, handle, _hint) = drive_engine(&dir);
let mut mirror = FsMirror::new();
let mut seen = 0usize;
pump_until(
&sent,
&handle,
&mut mirror,
&mut seen,
"link subtree",
|m| m.live.contains_key("link/inner/deep.txt"),
);
let link = &mirror.live["link"];
assert_eq!(link.entry_flags & FS_ENTRY_TYPE_MASK, FS_ENTRY_SYMLINK);
assert_ne!(
link.entry_flags & FS_ENTRY_LINK_DIR,
0,
"a symlinked directory must advertise that it can be expanded"
);
assert!(mirror.live.contains_key("link/top.txt"));
assert_eq!(
mirror.live["link/inner/deep.txt"].content.as_deref(),
Some(&b"payload"[..])
);
assert_eq!(mirror.live["tolink"].entry_flags & FS_ENTRY_LINK_DIR, 0);
assert_eq!(mirror.live["dangling"].entry_flags & FS_ENTRY_LINK_DIR, 0);
assert!(mirror.live.contains_key("real/loop"));
assert!(
!mirror.live.keys().any(|k| k.starts_with("real/loop/")),
"a link to an ancestor must not be descended: {:?}",
mirror.live.keys().collect::<Vec<_>>()
);
assert!(mirror.live.contains_key("link/loop"));
assert!(
!mirror.live.keys().any(|k| k.starts_with("link/loop/")),
"cycle detection must hold through a symlinked path too: {:?}",
mirror.live.keys().collect::<Vec<_>>()
);
handle.command(Command::Stop);
}
#[test]
fn single_root_validation() {
use blit_remote::fs::{FS_STATUS_NOT_FOUND, FS_STATUS_OTHER};
let dir = temp_dir();
let file = dir.join("f.txt");
fs::write(&file, b"x").unwrap();
assert_eq!(
validate_single_root(&file.to_string_lossy()).unwrap(),
file.canonicalize().unwrap()
);
let (status, _) = validate_single_root(&dir.to_string_lossy()).unwrap_err();
assert_eq!(status, FS_STATUS_OTHER, "directory root refused");
let (status, _) =
validate_single_root(&dir.join("missing.txt").to_string_lossy()).unwrap_err();
assert_eq!(status, FS_STATUS_NOT_FOUND);
let _ = fs::remove_dir_all(&dir);
}
fn copy_mtime(from: &Path, to: &Path) {
let status = std::process::Command::new("touch")
.arg("-r")
.arg(from)
.arg(to)
.status()
.expect("touch");
assert!(status.success(), "touch -r failed");
assert_eq!(
stat_meta(from).unwrap().mtime_ns,
stat_meta(to).unwrap().mtime_ns,
"mtimes must be identical for the test to mean anything"
);
}
#[test]
fn single_sync_same_stat_rewrite() {
let dir = temp_dir().canonicalize().unwrap();
let reference = dir.join("reference");
let file = dir.join("note.txt");
fs::write(&reference, b"").unwrap();
fs::write(&file, b"one").unwrap();
copy_mtime(&reference, &file);
let (sent, handle, hint) = drive_single_engine(&file);
let mut mirror = FsMirror::new();
let mut seen = 0usize;
pump_until(&sent, &handle, &mut mirror, &mut seen, "initial", |m| {
m.live
.get("")
.is_some_and(|n| n.content.as_deref() == Some(&b"one"[..]))
});
fs::write(&file, b"two").unwrap();
copy_mtime(&reference, &file);
hint.send(Hint::Dirty(file.clone()));
pump_until(
&sent,
&handle,
&mut mirror,
&mut seen,
"same-stat rewrite",
|m| {
m.live
.get("")
.is_some_and(|n| n.content.as_deref() == Some(&b"two"[..]))
},
);
handle.command(Command::Stop);
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn single_sync_write_through() {
let dir = temp_dir().canonicalize().unwrap();
let file = dir.join("doc.txt");
fs::write(&file, b"hello").unwrap();
let (sent, handle, _hint) = drive_single_engine(&file);
let mut mirror = FsMirror::new();
let mut seen = 0usize;
pump_until(&sent, &handle, &mut mirror, &mut seen, "initial", |m| {
m.live.contains_key("")
});
let base = mirror.live[""].hash;
assert_eq!(base, blake3_128(b"hello"));
let (s, h, _) = await_done(&handle, &sent, 1, write_req(1, "", base, 0, b"world"));
assert_eq!(s, FS_DONE_OK);
assert_eq!(h, blake3_128(b"world"));
assert_eq!(fs::read(&file).unwrap(), b"world");
pump_until(&sent, &handle, &mut mirror, &mut seen, "echo", |m| {
m.live
.get("")
.is_some_and(|n| n.hash == blake3_128(b"world"))
});
let (s, disk, _) = await_done(&handle, &sent, 2, write_req(2, "", base, 0, b"x"));
assert_eq!(s, FS_DONE_CONFLICT);
assert_eq!(disk, blake3_128(b"world"));
let (s, _, _) = await_done(&handle, &sent, 3, write_req(3, "other.txt", 0, 0, b"no"));
assert_eq!(s, FS_DONE_INVALID);
let (s, _, _) = await_done(&handle, &sent, 4, write_req(4, "", 0, 0, b"no"));
assert_eq!(s, FS_DONE_CONFLICT);
let (s, _, _) = await_done(
&handle,
&sent,
5,
Command::Op(OpReq {
nonce: 5,
op: FS_OP_REMOVE,
a: String::new(),
b: String::new(),
base: blake3_128(b"world"),
mode: 0,
flags: 0,
inflight: None,
}),
);
assert_eq!(s, FS_DONE_OK);
assert!(!file.exists());
pump_until(&sent, &handle, &mut mirror, &mut seen, "removed", |m| {
m.live.is_empty()
});
assert_eq!(count_closed(&sent), 0, "REMOVE of '' must not close");
handle.command(Command::Stop);
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn write_cas_semantics() {
let root = temp_dir().canonicalize().unwrap();
let (sent, handle, _hint) = drive_engine(&root);
let (s, hash, _) = await_done(&handle, &sent, 1, write_req(1, "a.txt", 0, 0, b"hello"));
assert_eq!(s, FS_DONE_OK);
assert_eq!(fs::read(root.join("a.txt")).unwrap(), b"hello");
assert_eq!(hash, blake3_128(b"hello"));
let (s, disk, _) = await_done(&handle, &sent, 2, write_req(2, "a.txt", 0, 0, b"x"));
assert_eq!(s, FS_DONE_CONFLICT);
assert_eq!(disk, hash, "conflict carries the live disk hash");
assert_eq!(fs::read(root.join("a.txt")).unwrap(), b"hello", "unchanged");
let (s, h2, _) = await_done(&handle, &sent, 3, write_req(3, "a.txt", hash, 0, b"world"));
assert_eq!(s, FS_DONE_OK);
assert_eq!(h2, blake3_128(b"world"));
assert_eq!(fs::read(root.join("a.txt")).unwrap(), b"world");
let (s, _, _) = await_done(&handle, &sent, 4, write_req(4, "a.txt", hash, 0, b"z"));
assert_eq!(s, FS_DONE_CONFLICT, "stale base rejected");
let (s, _, _) = await_done(
&handle,
&sent,
5,
write_req(5, "a.txt", 0, FS_WRITE_NO_CAS, b"forced"),
);
assert_eq!(s, FS_DONE_OK);
assert_eq!(fs::read(root.join("a.txt")).unwrap(), b"forced");
let (s, _, _) = await_done(
&handle,
&sent,
6,
write_req(6, "d/e/f.txt", 0, FS_WRITE_MKPARENTS, b"deep"),
);
assert_eq!(s, FS_DONE_OK);
assert_eq!(fs::read(root.join("d/e/f.txt")).unwrap(), b"deep");
handle.command(Command::Stop);
let _ = fs::remove_dir_all(&root);
}
fn delta_req(nonce: u16, path: &str, base: u128, flags: u8, ops: &[u8]) -> Command {
Command::Write(WriteReq {
nonce,
path: path.into(),
base,
mode: 0,
flags,
content_kind: blit_remote::fs::FS_WRITE_CONTENT_DELTA,
content: ops.to_vec(),
inflight: None,
})
}
#[test]
fn write_delta_applies_against_cas_base() {
let root = temp_dir().canonicalize().unwrap();
let (sent, handle, _hint) = drive_engine(&root);
let old = b"hello world".as_slice();
let (s, h1, _) = await_done(&handle, &sent, 1, write_req(1, "a.txt", 0, 0, old));
assert_eq!(s, FS_DONE_OK);
let new = b"hello brave world".as_slice();
let ops = encode_delta(old, new);
assert_eq!(
blit_remote::fs::apply_fs_delta(old, &ops).as_deref(),
Some(new)
);
let (s, h2, _) = await_done(&handle, &sent, 2, delta_req(2, "a.txt", h1, 0, &ops));
assert_eq!(s, FS_DONE_OK);
assert_eq!(h2, blake3_128(new));
assert_eq!(fs::read(root.join("a.txt")).unwrap(), new);
let (s, disk, _) = await_done(&handle, &sent, 3, delta_req(3, "a.txt", h1, 0, &ops));
assert_eq!(s, FS_DONE_CONFLICT, "stale delta base must conflict");
assert_eq!(disk, h2, "conflict carries the live disk hash");
assert_eq!(fs::read(root.join("a.txt")).unwrap(), new);
let (s, _, _) = await_done(
&handle,
&sent,
4,
delta_req(4, "a.txt", h2, FS_WRITE_NO_CAS, &ops),
);
assert_eq!(s, FS_DONE_INVALID);
let (s, _, _) = await_done(&handle, &sent, 5, delta_req(5, "a.txt", 0, 0, &ops));
assert_eq!(s, FS_DONE_INVALID);
let (s, _, _) = await_done(&handle, &sent, 6, delta_req(6, "a.txt", h2, 0, &[0xFF, 1]));
assert_eq!(s, FS_DONE_INVALID);
assert_eq!(fs::read(root.join("a.txt")).unwrap(), new);
let (s, disk, _) = await_done(&handle, &sent, 7, delta_req(7, "gone.txt", h2, 0, &ops));
assert_eq!(s, FS_DONE_CONFLICT);
assert_eq!(disk, 0);
let mut mirror = FsMirror::new();
let mut seen = 0usize;
pump_until(&sent, &handle, &mut mirror, &mut seen, "delta echo", |m| {
m.live.get("a.txt").is_some_and(|n| n.hash == h2)
});
handle.command(Command::Stop);
let _ = fs::remove_dir_all(&root);
}
#[test]
fn write_delta_on_single_sync() {
let dir = temp_dir().canonicalize().unwrap();
let file = dir.join("buf.txt");
fs::write(&file, b"alpha").unwrap();
let (sent, handle, _hint) = drive_single_engine(&file);
let mut mirror = FsMirror::new();
let mut seen = 0usize;
pump_until(&sent, &handle, &mut mirror, &mut seen, "initial", |m| {
m.live.contains_key("")
});
let h0 = mirror.live[""].hash;
let ops = encode_delta(b"alpha", b"alpha beta");
let (s, h1, _) = await_done(&handle, &sent, 1, delta_req(1, "", h0, 0, &ops));
assert_eq!(s, FS_DONE_OK);
assert_eq!(h1, blake3_128(b"alpha beta"));
assert_eq!(fs::read(&file).unwrap(), b"alpha beta");
let ops2 = encode_delta(b"alpha beta", b"alpha beta gamma");
let (s, h2, _) = await_done(&handle, &sent, 2, delta_req(2, "", h1, 0, &ops2));
assert_eq!(s, FS_DONE_OK);
assert_eq!(h2, blake3_128(b"alpha beta gamma"));
assert_eq!(fs::read(&file).unwrap(), b"alpha beta gamma");
let (s, _, _) = await_done(&handle, &sent, 3, delta_req(3, "x.txt", h2, 0, &ops2));
assert_eq!(s, FS_DONE_INVALID);
pump_until(&sent, &handle, &mut mirror, &mut seen, "echo", |m| {
m.live.get("").is_some_and(|n| n.hash == h2)
});
handle.command(Command::Stop);
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn write_refuses_traversal() {
let root = temp_dir().canonicalize().unwrap();
let sibling = root.parent().unwrap().join("blit-escape-victim.txt");
let _ = fs::remove_file(&sibling);
let (sent, handle, _hint) = drive_engine(&root);
for (i, p) in ["../blit-escape-victim.txt", "%2E%2E/blit-escape-victim.txt"]
.iter()
.enumerate()
{
let (s, _, _) = await_done(
&handle,
&sent,
i as u16 + 1,
write_req(i as u16 + 1, p, 0, 0, b"pwn"),
);
assert_eq!(s, FS_DONE_INVALID, "traversal {p} must be refused");
}
assert!(!sibling.exists(), "nothing escaped the root");
handle.command(Command::Stop);
let _ = fs::remove_dir_all(&root);
}
#[cfg(unix)]
#[test]
fn fetch_refuses_symlink_escape() {
let root = temp_dir().canonicalize().unwrap();
let secret = root.parent().unwrap().join("blit-fetch-secret.txt");
fs::write(&secret, b"top secret").unwrap();
std::os::unix::fs::symlink(root.parent().unwrap(), root.join("pub")).unwrap();
let (sent, handle, _hint) = drive_engine(&root);
let await_file = |nonce: u16| -> (u8, Vec<u8>) {
let deadline = Instant::now() + Duration::from_secs(5);
loop {
for msg in sent.lock().unwrap().iter() {
if msg[0] == blit_remote::fs::S2C_FS_FILE
&& let Some((n, status, data)) = blit_remote::fs::parse_fs_file(msg)
&& n == nonce
{
return (status, data.to_vec());
}
}
assert!(Instant::now() < deadline, "no FS_FILE for nonce {nonce}");
std::thread::sleep(Duration::from_millis(2));
}
};
handle.command(Command::Fetch {
nonce: 1,
path: "pub/blit-fetch-secret.txt".into(),
});
let (status, data) = await_file(1);
assert_ne!(status, FS_FILE_OK, "escape must be refused");
assert!(data.is_empty(), "no bytes leak past the confinement");
handle.command(Command::Stop);
let _ = fs::remove_file(&secret);
let _ = fs::remove_dir_all(&root);
}
#[test]
fn fs_ops_mkdir_rename_remove() {
let root = temp_dir().canonicalize().unwrap();
let (sent, handle, _hint) = drive_engine(&root);
let op = |nonce: u16, op: u8, a: &str, b: &str, base: u128, flags: u8| {
Command::Op(OpReq {
nonce,
op,
a: a.into(),
b: b.into(),
base,
mode: 0,
flags,
inflight: None,
})
};
let (s, _, _) = await_done(&handle, &sent, 1, op(1, FS_OP_MKDIR, "sub", "", 0, 0));
assert_eq!(s, FS_DONE_OK);
assert!(root.join("sub").is_dir());
let (s, _, _) = await_done(&handle, &sent, 2, op(2, FS_OP_MKDIR, "sub", "", 0, 0));
assert_eq!(s, FS_DONE_OK);
let (_, _, _) = await_done(&handle, &sent, 3, write_req(3, "sub/x.txt", 0, 0, b"hi"));
let (s, _, _) = await_done(
&handle,
&sent,
4,
op(4, FS_OP_RENAME, "sub/x.txt", "sub/y.txt", 0, 0),
);
assert_eq!(s, FS_DONE_OK);
assert!(!root.join("sub/x.txt").exists());
assert_eq!(fs::read(root.join("sub/y.txt")).unwrap(), b"hi");
let (s, _, _) = await_done(
&handle,
&sent,
5,
op(5, FS_OP_RENAME, "sub/gone.txt", "sub/z.txt", 0, 0),
);
assert_eq!(s, FS_DONE_NOT_FOUND);
let (s, _, _) = await_done(&handle, &sent, 6, op(6, FS_OP_REMOVE, "sub", "", 0, 0));
assert_eq!(s, FS_DONE_OK);
assert!(!root.join("sub").exists());
let (s, _, _) = await_done(&handle, &sent, 7, op(7, FS_OP_REMOVE, "sub", "", 0, 0));
assert_eq!(s, FS_DONE_NOT_FOUND);
handle.command(Command::Stop);
let _ = fs::remove_dir_all(&root);
}
#[cfg(unix)]
#[test]
fn fs_ops_symlink_hardlink() {
let root = temp_dir().canonicalize().unwrap();
let (sent, handle, hint) = drive_engine(&root);
let op = |nonce: u16, op: u8, a: &str, b: &str, base: u128, flags: u8| {
Command::Op(OpReq {
nonce,
op,
a: a.into(),
b: b.into(),
base,
mode: 0,
flags,
inflight: None,
})
};
let (s, h, _) = await_done(&handle, &sent, 1, op(1, FS_OP_SYMLINK, "a.txt", "ln", 0, 0));
assert_eq!(s, FS_DONE_OK);
assert_eq!(fs::read_link(root.join("ln")).unwrap(), Path::new("a.txt"));
assert_eq!(h, blake3_128(b"a.txt"));
let (s, disk, _) = await_done(&handle, &sent, 2, op(2, FS_OP_SYMLINK, "other", "ln", 0, 0));
assert_eq!(s, FS_DONE_CONFLICT);
assert_eq!(disk, h);
let (s, h2, _) = await_done(&handle, &sent, 3, op(3, FS_OP_SYMLINK, "b.txt", "ln", h, 0));
assert_eq!(s, FS_DONE_OK);
assert_eq!(h2, blake3_128(b"b.txt"));
assert_eq!(fs::read_link(root.join("ln")).unwrap(), Path::new("b.txt"));
let (s, _, _) = await_done(&handle, &sent, 4, op(4, FS_OP_SYMLINK, "c", "ln", h, 0));
assert_eq!(s, FS_DONE_CONFLICT);
let (s, _, _) = await_done(
&handle,
&sent,
5,
op(5, FS_OP_SYMLINK, "gone/dangling", "ln", 0, FS_OP_NO_CAS),
);
assert_eq!(s, FS_DONE_OK);
assert_eq!(
fs::read_link(root.join("ln")).unwrap(),
Path::new("gone/dangling")
);
fs::create_dir(root.join("d")).unwrap();
let (s, _, _) = await_done(
&handle,
&sent,
6,
op(6, FS_OP_SYMLINK, "x", "d", 0, FS_OP_NO_CAS),
);
assert_eq!(s, FS_DONE_WRONG_TYPE);
let (s, fh, _) = await_done(&handle, &sent, 10, write_req(10, "f.txt", 0, 0, b"hello"));
assert_eq!(s, FS_DONE_OK);
let (s, lh, _) = await_done(
&handle,
&sent,
11,
op(11, FS_OP_HARDLINK, "f.txt", "f2.txt", 0, 0),
);
assert_eq!(s, FS_DONE_OK);
assert_eq!(lh, fh);
assert_eq!(fs::read(root.join("f2.txt")).unwrap(), b"hello");
{
use std::os::unix::fs::MetadataExt;
assert_eq!(
fs::metadata(root.join("f.txt")).unwrap().ino(),
fs::metadata(root.join("f2.txt")).unwrap().ino()
);
}
let (s, _, _) = await_done(
&handle,
&sent,
12,
op(12, FS_OP_HARDLINK, "f.txt", "f2.txt", 0, 0),
);
assert_eq!(s, FS_DONE_CONFLICT);
let (s, _, _) = await_done(
&handle,
&sent,
13,
op(13, FS_OP_HARDLINK, "ln", "ln2", 0, 0),
);
assert_eq!(s, FS_DONE_WRONG_TYPE);
let (s, _, _) = await_done(
&handle,
&sent,
14,
op(14, FS_OP_HARDLINK, "nope", "n2", 0, 0),
);
assert_eq!(s, FS_DONE_NOT_FOUND);
std::os::unix::fs::symlink("ext-target", root.join("ext")).unwrap();
hint.send(Hint::Dirty(root.join("ext")));
let mut mirror = FsMirror::new();
let mut seen = 0usize;
let deadline = Instant::now() + Duration::from_secs(5);
loop {
for msg in sent.lock().unwrap().clone()[seen..].iter() {
seen += 1;
if msg[0] == blit_remote::fs::S2C_FS_UPDATE {
let id = mirror.apply_update(msg).unwrap();
handle.command(Command::Ack(id));
}
}
if mirror
.live
.get("ext")
.is_some_and(|n| n.content.as_deref() == Some(&b"ext-target"[..]))
&& mirror.live.contains_key("ln")
{
break;
}
assert!(Instant::now() < deadline, "symlink content never synced");
std::thread::sleep(Duration::from_millis(2));
}
let node = mirror.live.get("ext").unwrap();
assert_eq!(node.entry_flags & FS_ENTRY_TYPE_MASK, FS_ENTRY_SYMLINK);
assert_eq!(node.hash, blake3_128(b"ext-target"));
assert_eq!(node.size, "ext-target".len() as u64);
let own = mirror.live.get("ln").unwrap();
assert_eq!(own.entry_flags & FS_ENTRY_TYPE_MASK, FS_ENTRY_SYMLINK);
assert_eq!(own.hash, blake3_128(b"gone/dangling"));
handle.command(Command::Fetch {
nonce: 20,
path: "ln".into(),
});
let deadline = Instant::now() + Duration::from_secs(5);
'fetch: loop {
for msg in sent.lock().unwrap().iter() {
if msg[0] == blit_remote::fs::S2C_FS_FILE
&& let Some((20, status, data)) = blit_remote::fs::parse_fs_file(msg)
{
assert_eq!(status, FS_FILE_OK);
assert_eq!(data, b"gone/dangling");
break 'fetch;
}
}
assert!(Instant::now() < deadline, "no FS_FILE for the symlink");
std::thread::sleep(Duration::from_millis(2));
}
handle.command(Command::Stop);
let _ = fs::remove_dir_all(&root);
}
#[cfg(unix)]
#[test]
fn unreadable_content_recovers_when_readable() {
use std::os::unix::fs::PermissionsExt;
let root = temp_dir();
let file = root.join("secret.txt");
fs::write(&file, b"classified").unwrap();
fs::set_permissions(&file, fs::Permissions::from_mode(0o000)).unwrap();
if fs::read(&file).is_ok() {
let _ = fs::remove_dir_all(&root);
return;
}
let (sent, handle, _hint) = drive_engine(&root);
let mut mirror = FsMirror::new();
let mut acked = 0usize;
let pump = |mirror: &mut FsMirror, acked: &mut usize| {
for msg in sent.lock().unwrap().clone()[*acked..].iter() {
if msg[0] == blit_remote::fs::S2C_FS_UPDATE {
let id = mirror.apply_update(msg).unwrap();
handle.command(Command::Ack(id));
*acked += 1;
} else {
*acked += 1;
}
}
};
for _ in 0..200 {
pump(&mut mirror, &mut acked);
if let Some(node) = mirror.live.get("secret.txt")
&& node.entry_flags & FS_ENTRY_UNREADABLE != 0
{
break;
}
std::thread::sleep(Duration::from_millis(5));
}
let node = mirror.live.get("secret.txt").expect("file present");
assert_ne!(
node.entry_flags & FS_ENTRY_UNREADABLE,
0,
"expected UNREADABLE"
);
assert!(node.content.is_none());
fs::set_permissions(&file, fs::Permissions::from_mode(0o644)).unwrap();
let deadline = Instant::now() + Duration::from_secs(10);
loop {
pump(&mut mirror, &mut acked);
if mirror.live["secret.txt"].content.as_deref() == Some(&b"classified"[..]) {
break;
}
assert!(Instant::now() < deadline, "content never recovered");
std::thread::sleep(Duration::from_millis(5));
}
handle.command(Command::Stop);
let _ = fs::remove_dir_all(&root);
}
#[cfg(unix)]
#[test]
fn retry_survives_rename() {
use std::os::unix::fs::PermissionsExt;
let root = temp_dir();
let old = root.join("a.txt");
fs::write(&old, b"payload").unwrap();
fs::set_permissions(&old, fs::Permissions::from_mode(0o000)).unwrap();
if fs::read(&old).is_ok() {
let _ = fs::remove_dir_all(&root);
return;
}
let (sent, handle, hint_tx) = drive_engine(&root);
let mut mirror = FsMirror::new();
let mut acked = 0usize;
let pump = |mirror: &mut FsMirror, acked: &mut usize| {
for msg in sent.lock().unwrap().clone()[*acked..].iter() {
if msg[0] == blit_remote::fs::S2C_FS_UPDATE {
let id = mirror.apply_update(msg).unwrap();
handle.command(Command::Ack(id));
}
*acked += 1;
}
};
for _ in 0..200 {
pump(&mut mirror, &mut acked);
if mirror.live.contains_key("a.txt") {
break;
}
std::thread::sleep(Duration::from_millis(5));
}
assert!(mirror.live["a.txt"].content.is_none());
fs::set_permissions(&old, fs::Permissions::from_mode(0o644)).unwrap();
let new = root.join("b.txt");
fs::rename(&old, &new).unwrap();
hint_tx.send(Hint::Dirty(old));
hint_tx.send(Hint::Dirty(new));
let deadline = Instant::now() + Duration::from_secs(10);
loop {
pump(&mut mirror, &mut acked);
if mirror.live.get("b.txt").and_then(|n| n.content.as_deref()) == Some(&b"payload"[..])
{
break;
}
assert!(
Instant::now() < deadline,
"content did not follow the rename: {:?}",
mirror.live.get("b.txt")
);
std::thread::sleep(Duration::from_millis(5));
}
assert!(!mirror.live.contains_key("a.txt"));
handle.command(Command::Stop);
let _ = fs::remove_dir_all(&root);
}
#[test]
fn diff_plain_changes() {
let mut prev = Index::new();
prev.insert("".into(), meta(FS_ENTRY_DIR, 0, 0, 1));
prev.insert("a".into(), meta(FS_ENTRY_FILE, 1, 1, 2));
prev.insert("b".into(), meta(FS_ENTRY_FILE, 1, 1, 3));
let mut curr = Index::new();
curr.insert("".into(), meta(FS_ENTRY_DIR, 0, 0, 1));
curr.insert("a".into(), meta(FS_ENTRY_FILE, 2, 2, 2)); curr.insert("c".into(), meta(FS_ENTRY_FILE, 1, 1, 9)); let ops = diff(&prev, &curr);
assert!(ops.contains(&DiffOp::Delete { path: "b".into() }));
assert!(ops.contains(&DiffOp::Upsert {
path: "a".into(),
content_changed: true
}));
assert!(ops.contains(&DiffOp::Upsert {
path: "c".into(),
content_changed: true
}));
assert_eq!(ops.len(), 3);
}
#[test]
fn diff_mass_delete_prunes_to_ancestors() {
let mut prev = Index::new();
prev.insert("".into(), meta(FS_ENTRY_DIR, 0, 0, 1));
prev.insert("a".into(), meta(FS_ENTRY_DIR, 0, 0, 2));
prev.insert("a!x".into(), meta(FS_ENTRY_FILE, 1, 1, 3));
prev.insert("a/b".into(), meta(FS_ENTRY_DIR, 0, 0, 4));
prev.insert("a/b/c".into(), meta(FS_ENTRY_FILE, 1, 1, 5));
prev.insert("ab".into(), meta(FS_ENTRY_FILE, 1, 1, 6));
let mut curr = Index::new();
curr.insert("".into(), meta(FS_ENTRY_DIR, 0, 0, 1));
let mut deleted: Vec<String> = diff(&prev, &curr)
.into_iter()
.map(|op| match op {
DiffOp::Delete { path } => path,
other => panic!("unexpected {other:?}"),
})
.collect();
deleted.sort();
assert_eq!(deleted, ["a", "a!x", "ab"].map(String::from));
}
#[test]
fn diff_changed_matches_full_diff() {
let mut prev = Index::new();
prev.insert("".into(), meta(FS_ENTRY_DIR, 0, 0, 1));
prev.insert("d".into(), meta(FS_ENTRY_DIR, 0, 0, 2));
prev.insert("d/f".into(), meta(FS_ENTRY_FILE, 5, 10, 3));
prev.insert("gone".into(), meta(FS_ENTRY_DIR, 0, 0, 4));
prev.insert("gone/x".into(), meta(FS_ENTRY_FILE, 1, 1, 5));
prev.insert("same".into(), meta(FS_ENTRY_FILE, 2, 2, 6));
prev.insert("touched".into(), meta(FS_ENTRY_FILE, 3, 3, 7));
let mut curr = Index::new();
curr.insert("".into(), meta(FS_ENTRY_DIR, 0, 0, 1));
curr.insert("e".into(), meta(FS_ENTRY_DIR, 0, 0, 2));
curr.insert("e/f".into(), meta(FS_ENTRY_FILE, 5, 10, 3));
curr.insert("same".into(), meta(FS_ENTRY_FILE, 2, 2, 6));
curr.insert("touched".into(), meta(FS_ENTRY_FILE, 9, 9, 7));
curr.insert("new".into(), meta(FS_ENTRY_FILE, 1, 1, 8));
let changed: std::collections::BTreeSet<String> = [
"d", "d/f", "e", "e/f", "gone", "gone/x", "touched", "new", "same",
]
.into_iter()
.map(String::from)
.collect();
let full = diff(&prev, &curr);
assert_eq!(diff_changed(&prev, &curr, &changed), full);
assert!(full.contains(&DiffOp::Move {
from: "d".into(),
to: "e".into()
}));
assert!(full.contains(&DiffOp::Delete {
path: "gone".into()
}));
assert!(!full.iter().any(
|op| matches!(op, DiffOp::Upsert { path, .. } | DiffOp::Delete { path } if path == "same")
));
}
#[test]
fn retry_backoff_doubles_and_caps() {
let latency = Duration::from_millis(20);
assert_eq!(retry_backoff(1, latency), Duration::from_millis(20));
assert_eq!(retry_backoff(2, latency), Duration::from_millis(40));
assert_eq!(retry_backoff(5, latency), Duration::from_millis(320));
assert_eq!(retry_backoff(8, latency), Duration::from_secs(2));
assert_eq!(retry_backoff(64, latency), Duration::from_secs(2));
}
#[test]
fn engine_converges() {
let root = temp_dir();
fs::write(root.join("hello.txt"), b"hello").unwrap();
fs::create_dir(root.join("sub")).unwrap();
fs::write(root.join("sub/nested.txt"), b"nested").unwrap();
let shared = open_root_unwatched(test_key(&root));
let hint_tx = shared.hint_sender();
let sent: Arc<Mutex<Vec<Vec<u8>>>> = Default::default();
let sent2 = sent.clone();
let opts = SyncOptions {
content: true,
latency: Duration::from_millis(5),
..Default::default()
};
let handle = start_sync(
&shared,
7,
opts,
Box::new(move |msg| {
sent2.lock().unwrap().push(msg);
true
}),
);
let wait_updates = |min: usize| {
for _ in 0..200 {
if sent.lock().unwrap().len() >= min {
return;
}
std::thread::sleep(Duration::from_millis(5));
}
panic!("timed out waiting for {min} updates");
};
wait_updates(1);
let mut mirror = FsMirror::new();
let mut acked = 0usize;
let apply_all = |mirror: &mut FsMirror, acked: &mut usize| {
let msgs = sent.lock().unwrap().clone();
for msg in &msgs[*acked..] {
if msg[0] == blit_remote::fs::S2C_FS_UPDATE {
let id = mirror.apply_update(msg).expect("valid update");
handle.command(Command::Ack(id));
}
}
*acked = msgs.len();
};
apply_all(&mut mirror, &mut acked);
assert_eq!(
mirror.live["hello.txt"].content.as_deref(),
Some(&b"hello"[..])
);
assert_eq!(
mirror.live["sub/nested.txt"].content.as_deref(),
Some(&b"nested"[..])
);
assert!(mirror.live.contains_key("")); assert!(mirror.live.contains_key("sub"));
fs::write(root.join("hello.txt"), b"changed").unwrap();
fs::remove_file(root.join("sub/nested.txt")).unwrap();
fs::write(root.join("sub/other.txt"), b"other").unwrap();
hint_tx.send(Hint::Dirty(root.join("hello.txt")));
hint_tx.send(Hint::Dirty(root.join("sub")));
wait_updates(acked + 1);
std::thread::sleep(Duration::from_millis(30));
apply_all(&mut mirror, &mut acked);
assert_eq!(
mirror.live["hello.txt"].content.as_deref(),
Some(&b"changed"[..])
);
assert!(!mirror.live.contains_key("sub/nested.txt"));
assert_eq!(
mirror.live["sub/other.txt"].content.as_deref(),
Some(&b"other"[..])
);
fs::write(root.join("late.txt"), b"late").unwrap();
hint_tx.send(Hint::Rescan);
wait_updates(acked + 1);
std::thread::sleep(Duration::from_millis(30));
apply_all(&mut mirror, &mut acked);
assert_eq!(
mirror.live["late.txt"].content.as_deref(),
Some(&b"late"[..])
);
handle.command(Command::Stop);
let _ = fs::remove_dir_all(&root);
}
#[test]
fn snapshot_respects_ack_window() {
let root = temp_dir();
for i in 0..50 {
fs::write(root.join(format!("f{i:02}.txt")), vec![b'x'; 256]).unwrap();
}
let shared = open_root_unwatched(test_key(&root));
let sent: Arc<Mutex<Vec<Vec<u8>>>> = Default::default();
let sent2 = sent.clone();
let window = 2048usize;
let opts = SyncOptions {
content: true,
latency: Duration::from_millis(5),
window_bytes: window,
batch_target: 512,
..Default::default()
};
let handle = start_sync(
&shared,
3,
opts,
Box::new(move |msg| {
sent2.lock().unwrap().push(msg);
true
}),
);
let mut mirror = FsMirror::new();
let mut applied = 0usize;
let mut synced = false;
for _ in 0..400 {
std::thread::sleep(Duration::from_millis(5));
let msgs = sent.lock().unwrap().clone();
let outstanding: usize = msgs[applied..].iter().map(|m| m.len()).sum();
let max_update = msgs.iter().map(|m| m.len()).max().unwrap_or(0);
assert!(
outstanding <= window + max_update,
"engine outran the window: {outstanding} unacked bytes"
);
for msg in &msgs[applied..] {
if msg[0] == blit_remote::fs::S2C_FS_UPDATE {
let flags = msg[7];
let id = mirror.apply_update(msg).expect("valid update");
handle.command(Command::Ack(id));
if flags & FS_UPDATE_SYNC != 0 {
synced = true;
}
}
}
applied = msgs.len();
if synced {
break;
}
}
assert!(synced, "snapshot never reached SYNC");
assert_eq!(
mirror
.live
.iter()
.filter(|(_, n)| n.content.is_some())
.count(),
50
);
assert!(
applied > 5,
"expected a paced series, got {applied} updates"
);
handle.command(Command::Stop);
let _ = fs::remove_dir_all(&root);
}
#[test]
fn native_backend_delivers_changes() {
let root = temp_dir().canonicalize().unwrap();
fs::write(root.join("seed.txt"), b"seed").unwrap();
let shared = open_root(test_key(&root)).expect("arm native watch");
let sent: Arc<Mutex<Vec<Vec<u8>>>> = Default::default();
let sent2 = sent.clone();
let opts = SyncOptions {
content: true,
latency: Duration::from_millis(5),
..Default::default()
};
let handle = start_sync(
&shared,
9,
opts,
Box::new(move |msg| {
sent2.lock().unwrap().push(msg);
true
}),
);
let mut mirror = FsMirror::new();
let mut applied = 0usize;
let apply_all = |mirror: &mut FsMirror, applied: &mut usize| {
let msgs = sent.lock().unwrap().clone();
for msg in &msgs[*applied..] {
if msg[0] == blit_remote::fs::S2C_FS_UPDATE {
let id = mirror.apply_update(msg).expect("valid update");
handle.command(Command::Ack(id));
}
}
*applied = msgs.len();
};
for _ in 0..200 {
apply_all(&mut mirror, &mut applied);
if mirror.live.contains_key("seed.txt") {
break;
}
std::thread::sleep(Duration::from_millis(5));
}
assert!(mirror.live.contains_key("seed.txt"));
fs::create_dir(root.join("dir")).unwrap();
fs::write(root.join("dir/new.txt"), b"native").unwrap();
let deadline = Instant::now() + Duration::from_secs(10);
loop {
apply_all(&mut mirror, &mut applied);
if mirror
.live
.get("dir/new.txt")
.is_some_and(|n| n.content.as_deref() == Some(b"native"))
{
break;
}
assert!(
Instant::now() < deadline,
"native backend never delivered the change; live = {:?}",
mirror.live.keys().collect::<Vec<_>>()
);
std::thread::sleep(Duration::from_millis(10));
}
handle.command(Command::Stop);
let _ = fs::remove_dir_all(&root);
}
#[test]
fn single_native_backend_follows_file() {
let dir = temp_dir().canonicalize().unwrap();
let file = dir.join("watched.txt");
fs::write(&file, b"one").unwrap();
let shared = open_single_root(file.clone()).expect("arm native watch on parent");
assert!(shared.is_single());
let sent: Arc<Mutex<Vec<Vec<u8>>>> = Default::default();
let sent2 = sent.clone();
let opts = SyncOptions {
content: true,
recursive: false,
latency: Duration::from_millis(5),
..Default::default()
};
let handle = start_sync(
&shared,
15,
opts,
Box::new(move |msg| {
sent2.lock().unwrap().push(msg);
true
}),
);
let mut mirror = FsMirror::new();
let mut seen = 0usize;
pump_until(&sent, &handle, &mut mirror, &mut seen, "initial", |m| {
m.live
.get("")
.is_some_and(|n| n.content.as_deref() == Some(&b"one"[..]))
});
assert_eq!(mirror.live.len(), 1);
fs::write(&file, b"two").unwrap();
pump_until(
&sent,
&handle,
&mut mirror,
&mut seen,
"native modify",
|m| {
m.live
.get("")
.is_some_and(|n| n.content.as_deref() == Some(&b"two"[..]))
},
);
fs::remove_file(&file).unwrap();
pump_until(
&sent,
&handle,
&mut mirror,
&mut seen,
"native delete",
|m| m.live.is_empty(),
);
assert_eq!(count_closed(&sent), 0, "delete must not close the sync");
fs::write(&file, b"three").unwrap();
pump_until(
&sent,
&handle,
&mut mirror,
&mut seen,
"native recreate",
|m| {
m.live
.get("")
.is_some_and(|n| n.content.as_deref() == Some(&b"three"[..]))
},
);
handle.command(Command::Stop);
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn property_random_mutations_converge() {
for seed in [1u64, 7, 42, 0xdead_beef] {
property_run(seed);
}
}
fn xorshift(state: &mut u64) -> u64 {
*state ^= *state << 13;
*state ^= *state >> 7;
*state ^= *state << 17;
*state
}
fn scan_disk(root: &Path) -> BTreeMap<String, Option<Vec<u8>>> {
fn walk(map: &mut BTreeMap<String, Option<Vec<u8>>>, abs: &Path, rel: &str) {
let Ok(md) = fs::symlink_metadata(abs) else {
return;
};
if md.is_dir() {
map.insert(rel.to_string(), None);
let Ok(entries) = fs::read_dir(abs) else {
return;
};
for entry in entries.flatten() {
let name = entry.file_name().to_string_lossy().into_owned();
let child_rel = if rel.is_empty() {
name.clone()
} else {
format!("{rel}/{name}")
};
walk(map, &entry.path(), &child_rel);
}
} else if md.is_file() {
map.insert(rel.to_string(), fs::read(abs).ok());
}
}
let mut map = BTreeMap::new();
walk(&mut map, root, "");
map
}
fn mirror_state(mirror: &FsMirror) -> BTreeMap<String, Option<Vec<u8>>> {
mirror
.live
.iter()
.map(|(path, node)| {
let content = if node.entry_flags & FS_ENTRY_TYPE_MASK == FS_ENTRY_FILE {
node.content.clone()
} else {
None
};
(path.clone(), content)
})
.collect()
}
struct PropClient {
sent: Arc<Mutex<Vec<Vec<u8>>>>,
handle: SyncHandle,
mirror: FsMirror,
applied: usize,
highest_unacked: Option<u32>,
}
impl PropClient {
fn start(shared: &Arc<SharedRootHandle>, sync_id: u16) -> Self {
let sent: Arc<Mutex<Vec<Vec<u8>>>> = Default::default();
let sent2 = sent.clone();
let opts = SyncOptions {
content: true,
latency: Duration::from_millis(3),
window_bytes: 4096,
batch_target: 1024,
..Default::default()
};
let handle = start_sync(
shared,
sync_id,
opts,
Box::new(move |msg| {
sent2.lock().unwrap().push(msg);
true
}),
);
PropClient {
sent,
handle,
mirror: FsMirror::new(),
applied: 0,
highest_unacked: None,
}
}
fn pump(&mut self, rng: &mut u64, flush: bool) {
use blit_remote::fs::S2C_FS_UPDATE;
let msgs = self.sent.lock().unwrap().clone();
for msg in &msgs[self.applied..] {
if msg[0] == S2C_FS_UPDATE {
let id = self.mirror.apply_update(msg).expect("valid update");
self.highest_unacked = Some(id);
}
}
self.applied = msgs.len();
if let Some(id) = self.highest_unacked
&& (flush || xorshift(rng).is_multiple_of(2))
{
self.handle.command(Command::Ack(id));
self.highest_unacked = None;
}
}
}
fn property_run(seed: u64) {
let root = temp_dir();
let shared = open_root_unwatched(test_key(&root));
let hint_tx = shared.hint_sender();
let mut clients = [
PropClient::start(&shared, 11),
PropClient::start(&shared, 12),
];
let mut rng = seed | 1;
let dirs = ["", "d0", "d1", "d0/d2"];
let names = ["f0", "f1", "f2", "f3"];
for _round in 0..25 {
let mutations = 1 + xorshift(&mut rng) % 3;
for _ in 0..mutations {
let dir = dirs[(xorshift(&mut rng) % dirs.len() as u64) as usize];
let name = names[(xorshift(&mut rng) % names.len() as u64) as usize];
let rel: PathBuf = if dir.is_empty() {
name.into()
} else {
Path::new(dir).join(name)
};
let abs = root.join(&rel);
match xorshift(&mut rng) % 5 {
0 | 1 => {
let _ = fs::create_dir_all(abs.parent().unwrap());
let len = (xorshift(&mut rng) % 64) as usize;
let byte = (xorshift(&mut rng) & 0xFF) as u8;
let _ = fs::write(&abs, vec![byte; len]);
}
2 => {
let _ = fs::create_dir_all(&abs);
}
3 => {
if abs.is_dir() {
let _ = fs::remove_dir_all(&abs);
} else {
let _ = fs::remove_file(&abs);
}
}
_ => {
let target = abs.with_file_name(
names[(xorshift(&mut rng) % names.len() as u64) as usize],
);
if target != abs {
let _ = fs::rename(&abs, &target);
hint_tx.send(Hint::Dirty(target));
}
}
}
hint_tx.send(Hint::Dirty(abs.clone()));
hint_tx.send(Hint::Dirty(abs.parent().unwrap().to_path_buf()));
}
if xorshift(&mut rng).is_multiple_of(16) {
hint_tx.send(Hint::Rescan);
}
for client in &mut clients {
client.pump(&mut rng, false);
}
std::thread::sleep(Duration::from_millis(xorshift(&mut rng) % 8));
}
let disk = scan_disk(&root);
let deadline = Instant::now() + Duration::from_secs(30);
loop {
for client in &mut clients {
client.pump(&mut rng, true);
}
if clients
.iter()
.all(|client| mirror_state(&client.mirror) == disk)
{
break;
}
assert!(
Instant::now() < deadline,
"seed {seed}: mirrors never converged\n first: {:?}\n second: {:?}\n disk: {:?}",
mirror_state(&clients[0].mirror).keys().collect::<Vec<_>>(),
mirror_state(&clients[1].mirror).keys().collect::<Vec<_>>(),
disk.keys().collect::<Vec<_>>(),
);
std::thread::sleep(Duration::from_millis(5));
}
for client in &clients {
client.handle.command(Command::Stop);
}
let _ = fs::remove_dir_all(&root);
}
#[test]
fn shared_root_serves_multiple_clients() {
let root = temp_dir();
fs::write(root.join("a.txt"), b"alpha").unwrap();
let shared = open_root_unwatched(test_key(&root));
let joined = open_root_unwatched(test_key(&root));
assert!(Arc::ptr_eq(&shared, &joined));
let hint_tx = shared.hint_sender();
let start = |sync_id: u16| {
let sent: Arc<Mutex<Vec<Vec<u8>>>> = Default::default();
let sent2 = sent.clone();
let opts = SyncOptions {
content: true,
latency: Duration::from_millis(5),
..Default::default()
};
let handle = start_sync(
&shared,
sync_id,
opts,
Box::new(move |msg| {
sent2.lock().unwrap().push(msg);
true
}),
);
(sent, handle)
};
let (sent_a, handle_a) = start(21);
let (sent_b, handle_b) = start(22);
let converge = |sent: &Arc<Mutex<Vec<Vec<u8>>>>,
handle: &SyncHandle,
mirror: &mut FsMirror,
applied: &mut usize,
path: &str,
want: &[u8]| {
for _ in 0..400 {
let msgs = sent.lock().unwrap().clone();
for msg in &msgs[*applied..] {
if msg[0] == blit_remote::fs::S2C_FS_UPDATE {
let id = mirror.apply_update(msg).expect("valid update");
handle.command(Command::Ack(id));
}
}
*applied = msgs.len();
if mirror
.live
.get(path)
.is_some_and(|n| n.content.as_deref() == Some(want))
{
return;
}
std::thread::sleep(Duration::from_millis(5));
}
panic!("mirror never saw {path}");
};
let mut mirror_a = FsMirror::new();
let mut mirror_b = FsMirror::new();
let (mut applied_a, mut applied_b) = (0usize, 0usize);
converge(
&sent_a,
&handle_a,
&mut mirror_a,
&mut applied_a,
"a.txt",
b"alpha",
);
converge(
&sent_b,
&handle_b,
&mut mirror_b,
&mut applied_b,
"a.txt",
b"alpha",
);
fs::write(root.join("b.txt"), b"beta").unwrap();
hint_tx.send(Hint::Dirty(root.join("b.txt")));
converge(
&sent_a,
&handle_a,
&mut mirror_a,
&mut applied_a,
"b.txt",
b"beta",
);
converge(
&sent_b,
&handle_b,
&mut mirror_b,
&mut applied_b,
"b.txt",
b"beta",
);
handle_a.command(Command::Stop);
handle_b.command(Command::Stop);
let _ = fs::remove_dir_all(&root);
}
#[test]
fn delta_roundtrips_through_client_apply() {
use blit_remote::fs::apply_fs_delta;
let cases: &[(&[u8], &[u8])] = &[
(b"hello world", b"hello world and more"), (b"hello world", b"say: hello world"), (b"hello cruel world", b"hello kind world"), (b"hello world", b"hello"), (b"hello", b"goodbye"), (b"", b"from nothing"), (b"to nothing", b""), (b"same", b"same"), ];
for (base, new) in cases {
let ops = encode_delta(base, new);
assert_eq!(
apply_fs_delta(base, &ops).as_deref(),
Some(*new),
"case {:?} -> {:?}",
base,
new
);
}
let base = vec![b'x'; 10_000];
let mut new = base.clone();
new.extend_from_slice(b"tail");
let ops = encode_delta(&base, &new);
assert!(
ops.len() < 20,
"append delta should be tiny, got {}",
ops.len()
);
assert_eq!(apply_fs_delta(&base, &ops).unwrap(), new);
}
#[test]
fn blob_store_lru_eviction() {
let mut store = BlobStore::new(1000);
let blob = |b: u8| Arc::new(vec![b; 400]);
store.put(1, blob(1));
store.put(2, blob(2));
store.get(1); store.put(3, blob(3)); assert!(store.get(2).is_none());
assert!(store.get(1).is_some());
assert!(store.get(3).is_some());
store.put(4, Arc::new(vec![0; 2000]));
assert!(store.get(4).is_none());
}
#[test]
fn engine_sends_deltas() {
use blit_remote::fs::{FsContent, FsRecord, fs_records, fs_update_records};
let root = temp_dir();
let big = vec![b'x'; 4096];
fs::write(root.join("log.txt"), &big).unwrap();
let shared = open_root_unwatched(test_key(&root));
let hint_tx = shared.hint_sender();
let sent: Arc<Mutex<Vec<Vec<u8>>>> = Default::default();
let sent2 = sent.clone();
let opts = SyncOptions {
content: true,
latency: Duration::from_millis(5),
..Default::default()
};
let handle = start_sync(
&shared,
13,
opts,
Box::new(move |msg| {
sent2.lock().unwrap().push(msg);
true
}),
);
let mut mirror = FsMirror::new();
let mut applied = 0usize;
let mut kinds: Vec<(String, &'static str)> = Vec::new();
let apply_all = |mirror: &mut FsMirror,
applied: &mut usize,
kinds: &mut Vec<(String, &'static str)>| {
let msgs = sent.lock().unwrap().clone();
for msg in &msgs[*applied..] {
if msg[0] == blit_remote::fs::S2C_FS_UPDATE {
let records = fs_update_records(msg).expect("decompress");
for record in fs_records(&records) {
if let FsRecord::Upsert { path, content, .. } = record {
let kind = match content {
FsContent::None => "none",
FsContent::Full(_) => "full",
FsContent::Delta(_) => "delta",
};
kinds.push((path.to_string(), kind));
}
}
let id = mirror.apply_update(msg).expect("valid update");
handle.command(Command::Ack(id));
}
}
*applied = msgs.len();
};
let wait_for = |sent: &Arc<Mutex<Vec<Vec<u8>>>>, min: usize| {
for _ in 0..400 {
if sent.lock().unwrap().len() >= min {
std::thread::sleep(Duration::from_millis(20));
return;
}
std::thread::sleep(Duration::from_millis(5));
}
panic!("timed out waiting for {min} messages");
};
wait_for(&sent, 1);
apply_all(&mut mirror, &mut applied, &mut kinds);
assert!(kinds.contains(&("log.txt".into(), "full")));
assert_eq!(mirror.live["log.txt"].content.as_deref(), Some(&big[..]));
kinds.clear();
let mut appended = big.clone();
appended.extend_from_slice(b"appended tail");
fs::write(root.join("log.txt"), &appended).unwrap();
hint_tx.send(Hint::Dirty(root.join("log.txt")));
wait_for(&sent, applied + 1);
apply_all(&mut mirror, &mut applied, &mut kinds);
assert!(
kinds.contains(&("log.txt".into(), "delta")),
"expected a delta record, got {kinds:?}"
);
assert_eq!(
mirror.live["log.txt"].content.as_deref(),
Some(&appended[..])
);
kinds.clear();
std::thread::sleep(Duration::from_millis(10)); fs::write(root.join("log.txt"), &appended).unwrap();
hint_tx.send(Hint::Dirty(root.join("log.txt")));
wait_for(&sent, applied + 1);
apply_all(&mut mirror, &mut applied, &mut kinds);
assert!(
kinds.contains(&("log.txt".into(), "none")),
"expected metadata-only, got {kinds:?}"
);
assert_eq!(
mirror.live["log.txt"].content.as_deref(),
Some(&appended[..])
);
assert_eq!(
mirror.live["log.txt"].entry_flags & blit_remote::fs::FS_ENTRY_NO_CONTENT,
0
);
handle.command(Command::Stop);
let _ = fs::remove_dir_all(&root);
}
#[test]
fn read_verified_stable() {
let root = temp_dir();
let f = root.join("x");
fs::write(&f, b"stable").unwrap();
match read_verified(&f) {
ReadOutcome::Stable(data) => assert_eq!(data, b"stable"),
_ => panic!("expected stable read"),
}
match read_verified(&root.join("missing")) {
ReadOutcome::Unreadable => {}
_ => panic!("expected unreadable"),
}
let _ = fs::remove_dir_all(&root);
}
}