Skip to main content

sapphire_framework_sync/
replica.rs

1//! A replica: one workspace root kept in sync with peers by joining path states.
2
3use std::collections::{BTreeMap, BTreeSet, HashMap};
4use std::fs;
5use std::io::ErrorKind;
6use std::path::{Path, PathBuf};
7use std::sync::Arc;
8use std::time::{Duration, SystemTime, UNIX_EPOCH};
9
10use grain_id::GrainId;
11
12use crate::entry::{Content, Entry, PathUpdate};
13use crate::error::{Error, Result};
14use crate::filter::SyncFilter;
15use crate::hash::ContentHash;
16use crate::hlc::Clock;
17use crate::id::ReplicaId;
18use crate::merge;
19use crate::paths;
20use crate::report::{Conflict, PauseReason, Report, ScanOutcome, SkipReason, Skipped};
21use crate::staging::{self, Stager};
22use crate::state::{DiskState, PathState};
23use crate::store::{Meta, ReplicaStore};
24use crate::vv::{Dot, VersionVector};
25
26/// Files larger than this are not synced unless configured otherwise.
27pub const DEFAULT_MAX_FILE_SIZE: u64 = 1024 * 1024 * 1024;
28
29/// A file whose mtime is this close to when it was checked is re-hashed.
30const RACY_NS: i64 = 2_000_000_000;
31
32/// Path states held in memory before they are written in one transaction. A redb
33/// commit costs several fsyncs, so writing each path on its own made a first sync
34/// slow enough on a slow disk to outlast a peer's idle timeout (issue #305).
35const MAX_PENDING: usize = 256;
36
37/// Where a replica gets content it does not have: a peer, in practice.
38///
39/// Content is handed over by path so that it is never held in memory whole.
40pub trait ContentSource {
41    /// A file holding exactly the content `hash`, if there is one. The replica copies
42    /// it through a verifying staging writer, so a wrong file is refused, not trusted.
43    fn locate(&self, hash: &ContentHash) -> Option<PathBuf>;
44}
45
46/// How a replica is set up.
47#[derive(Clone, Debug)]
48pub struct ReplicaConfig {
49    pub app_name: String,
50    pub root: PathBuf,
51    /// Recorded as `Entry::author` on local writes.
52    pub device_id: GrainId,
53    pub max_file_size: u64,
54    pub store_path: PathBuf,
55    pub staging_dir: PathBuf,
56}
57
58impl ReplicaConfig {
59    /// A config keeping the store and staging directory under `state_dir`.
60    pub fn new(
61        app_name: impl Into<String>,
62        root: impl Into<PathBuf>,
63        device_id: GrainId,
64        state_dir: &Path,
65    ) -> Self {
66        Self {
67            app_name: app_name.into(),
68            root: root.into(),
69            device_id,
70            max_file_size: DEFAULT_MAX_FILE_SIZE,
71            store_path: state_dir.join("sync.redb"),
72            staging_dir: state_dir.join("staging"),
73        }
74    }
75}
76
77/// One workspace root and its replica store.
78pub struct Replica {
79    config: ReplicaConfig,
80    store: ReplicaStore,
81    meta: Meta,
82    clock: Arc<dyn Clock>,
83    /// Path states committed but not yet written to the store; see [`Self::flush`].
84    /// Every read of a state goes through [`Self::get_state`], which sees these first.
85    pending: BTreeMap<String, PathState>,
86    /// Whether `pending` holds a state with newly joined versions. Such a state is
87    /// written before the workspace changes, so that a file placed from it is never
88    /// newer than what the store knows: a crash would otherwise leave a peer's content
89    /// on disk that the next scan records as a local edit.
90    pending_versions: bool,
91    #[cfg(any(test, feature = "test-util"))]
92    fault: Option<crate::testing::FaultPoint>,
93    /// Fires once after the next path a scan reconciles. Lets a test change the
94    /// filesystem in the middle of a scan.
95    ///
96    /// `Send` so that a replica held across a `tokio::spawn` still is: the app server keeps
97    /// one behind a mutex and moves it into a task per session.
98    #[cfg(any(test, feature = "test-util"))]
99    reconcile_hook: Option<Box<dyn FnOnce() + Send>>,
100}
101
102fn now_ns() -> i64 {
103    SystemTime::now()
104        .duration_since(UNIX_EPOCH)
105        .map(|d| i64::try_from(d.as_nanos()).unwrap_or(i64::MAX))
106        .unwrap_or(0)
107}
108
109fn stamp_of(meta: &fs::Metadata) -> (i64, u64) {
110    let mtime = meta
111        .modified()
112        .ok()
113        .and_then(|t| t.duration_since(UNIX_EPOCH).ok())
114        .map(|d| i64::try_from(d.as_nanos()).unwrap_or(i64::MAX))
115        .unwrap_or(0);
116    (mtime, meta.len())
117}
118
119/// How long staged content is kept whatever it holds; see [`Replica::collect_staging`].
120pub const STAGING_GRACE: Duration = Duration::from_secs(60 * 60);
121
122fn is_racy(disk: &DiskState) -> bool {
123    disk.mtime_ns.saturating_add(RACY_NS) >= disk.checked_ns
124}
125
126/// Move a verified staged file into place, falling back to copy + rename when the
127/// staging directory is on another volume.
128///
129/// The placement is durable when this returns: the caller commits the path's state next,
130/// and a committed state whose file a crash then lost would read as a local delete on
131/// the next scan — and spread to every peer as one.
132fn place(staged: &Path, dest: &Path) -> Result<()> {
133    if let Some(parent) = dest.parent() {
134        fs::create_dir_all(parent)?;
135    }
136    match fs::rename(staged, dest) {
137        Ok(()) => return sync_parent(dest),
138        Err(e) if e.kind() == ErrorKind::CrossesDevices => {}
139        Err(e) => return Err(e.into()),
140    }
141    let name = dest
142        .file_name()
143        .map(|n| n.to_string_lossy().into_owned())
144        .unwrap_or_default();
145    let tmp = dest.with_file_name(format!(".{name}.sapphire-tmp"));
146    let copied = fs::copy(staged, &tmp)
147        .and_then(|_| fs::File::open(&tmp)?.sync_all())
148        .and_then(|()| fs::rename(&tmp, dest));
149    if let Err(e) = copied {
150        let _ = fs::remove_file(&tmp);
151        return Err(e.into());
152    }
153    sync_parent(dest)?;
154    // The file is in place; the staged copy is a cache. Failing to remove it must not
155    // turn a completed placement into an error and leave the state uncommitted.
156    let _ = fs::remove_file(staged);
157    Ok(())
158}
159
160/// Make a rename into `path`'s directory durable. Unix only: std cannot open a
161/// directory to sync on Windows, where this is left to the filesystem.
162fn sync_parent(path: &Path) -> Result<()> {
163    #[cfg(unix)]
164    if let Some(parent) = path.parent() {
165        fs::File::open(parent)?.sync_all()?;
166    }
167    #[cfg(not(unix))]
168    let _ = path;
169    Ok(())
170}
171
172/// Lookups over the stored path states that would otherwise be a full store read per
173/// path: `case_twin` on a case-insensitive filesystem runs for every path of a first
174/// scan, and `content_path` runs for every file materialized from a peer. Built once
175/// per `scan`/`apply`/`fetch_missing` and kept current as states are committed, so it
176/// never goes stale within a call.
177#[derive(Debug, Default)]
178struct StoreIndex {
179    /// `disk.hash` of each stored path that has a file on disk.
180    hash_of: HashMap<String, ContentHash>,
181    /// `disk.hash` -> the stored paths whose file on disk has that hash, in path order.
182    by_hash: HashMap<ContentHash, BTreeSet<String>>,
183    /// Lowercased path -> the stored paths with that lowercase form. Left empty on a
184    /// case-sensitive filesystem, where no path can have a case twin.
185    by_lower: HashMap<String, BTreeSet<String>>,
186}
187
188impl StoreIndex {
189    fn build(states: &[(String, PathState)]) -> Self {
190        let mut index = Self::default();
191        for (rel, state) in states {
192            index.record(rel, state.disk.hash);
193        }
194        index
195    }
196
197    /// Note that `rel` now has `hash` on disk, replacing whatever it had before.
198    fn record(&mut self, rel: &str, hash: Option<ContentHash>) {
199        if paths::CASE_INSENSITIVE_FS {
200            self.by_lower
201                .entry(rel.to_lowercase())
202                .or_default()
203                .insert(rel.to_owned());
204        }
205        let previous = match hash {
206            Some(hash) => self.hash_of.insert(rel.to_owned(), hash),
207            None => self.hash_of.remove(rel),
208        };
209        if let Some(previous) = previous
210            && Some(previous) != hash
211            && let Some(paths) = self.by_hash.get_mut(&previous)
212        {
213            paths.remove(rel);
214            if paths.is_empty() {
215                self.by_hash.remove(&previous);
216            }
217        }
218        if let Some(hash) = hash {
219            self.by_hash.entry(hash).or_default().insert(rel.to_owned());
220        }
221    }
222
223    /// Another stored path, differing from `rel` only in case, whose file is on disk.
224    /// Always `None` on a case-sensitive filesystem.
225    fn case_twin(&self, rel: &str) -> Option<String> {
226        self.by_lower
227            .get(&rel.to_lowercase())?
228            .iter()
229            .find(|other| other.as_str() != rel && self.hash_of.contains_key(*other))
230            .cloned()
231    }
232
233    /// Stored paths whose file on disk has `hash`, in path order.
234    fn paths_with(&self, hash: &ContentHash) -> impl Iterator<Item = &String> {
235        self.by_hash.get(hash).into_iter().flatten()
236    }
237}
238
239/// Turn a per-path `Error::Io` into a `Skipped` entry and let the caller continue;
240/// propagate every other error so callers still abort on store/format/pause failures.
241fn tolerate_io(result: Result<()>, rel: &str, report: &mut Report) -> Result<()> {
242    match result {
243        Err(Error::Io(e)) => {
244            report.skipped.push(Skipped {
245                path: rel.to_owned(),
246                reason: SkipReason::Io(e.to_string()),
247            });
248            Ok(())
249        }
250        other => other,
251    }
252}
253
254impl Replica {
255    pub fn open(config: ReplicaConfig, clock: Arc<dyn Clock>) -> Result<Self> {
256        Self::open_inner(config, clock, None)
257    }
258
259    fn open_inner(
260        config: ReplicaConfig,
261        clock: Arc<dyn Clock>,
262        id: Option<ReplicaId>,
263    ) -> Result<Self> {
264        fs::create_dir_all(&config.staging_dir)?;
265        staging::clear_partials(&config.staging_dir)?;
266        let root = config.root.to_string_lossy().into_owned();
267        let store = ReplicaStore::open_with_id(&config.store_path, &root, id)?;
268        let meta = store.meta()?;
269        let replica = Self {
270            config,
271            store,
272            meta,
273            clock,
274            pending: BTreeMap::new(),
275            pending_versions: false,
276            #[cfg(any(test, feature = "test-util"))]
277            fault: None,
278            #[cfg(any(test, feature = "test-util"))]
279            reconcile_hook: None,
280        };
281        replica.collect_staging_quietly();
282        Ok(replica)
283    }
284
285    pub fn replica_id(&self) -> ReplicaId {
286        self.meta.replica_id
287    }
288
289    pub fn vv(&self) -> &VersionVector {
290        &self.meta.vv
291    }
292
293    pub fn config(&self) -> &ReplicaConfig {
294        &self.config
295    }
296
297    pub fn state(&self, path: &str) -> Result<Option<PathState>> {
298        self.get_state(path)
299    }
300
301    pub fn states(&self) -> Result<Vec<(String, PathState)>> {
302        self.all_states()
303    }
304
305    /// The state of `path`, pending or stored.
306    fn get_state(&self, path: &str) -> Result<Option<PathState>> {
307        match self.pending.get(path) {
308            Some(state) => Ok(Some(state.clone())),
309            None => self.store.get(path),
310        }
311    }
312
313    /// Every path state, pending or stored, in path order.
314    fn all_states(&self) -> Result<Vec<(String, PathState)>> {
315        let mut states = self.store.all()?;
316        if self.pending.is_empty() {
317            return Ok(states);
318        }
319        let mut pending = self.pending.clone();
320        for (rel, state) in &mut states {
321            if let Some(newer) = pending.remove(rel) {
322                *state = newer;
323            }
324        }
325        states.extend(pending);
326        states.sort_by(|a, b| a.0.cmp(&b.0));
327        Ok(states)
328    }
329
330    /// Write the pending path states, and the metadata, in one transaction.
331    fn flush(&mut self) -> Result<()> {
332        if self.pending.is_empty() {
333            return Ok(());
334        }
335        let states: Vec<(String, PathState)> = self
336            .pending
337            .iter()
338            .map(|(rel, state)| (rel.clone(), state.clone()))
339            .collect();
340        self.store.commit(&self.meta, &states)?;
341        self.pending.clear();
342        self.pending_versions = false;
343        Ok(())
344    }
345
346    /// Flush before the workspace changes if a pending state holds joined versions.
347    fn flush_versions(&mut self) -> Result<()> {
348        if self.pending_versions {
349            self.flush()?;
350        }
351        Ok(())
352    }
353
354    /// Flush what `result`'s call left pending, whether or not it failed: the files it
355    /// already placed must be recorded. The call's own error takes precedence.
356    fn finish<T>(&mut self, result: Result<T>) -> Result<T> {
357        let flushed = self.flush();
358        let value = result?;
359        flushed?;
360        Ok(value)
361    }
362
363    fn filter(&self) -> Result<SyncFilter> {
364        SyncFilter::load(&self.config.root, &self.config.app_name)
365    }
366
367    /// Why this replica must not scan or apply right now. A root or marker that is
368    /// missing while files were written would otherwise read as "everything deleted".
369    pub fn pause_reason(&self) -> Result<Option<PauseReason>> {
370        let root_ok = self.config.root.is_dir();
371        let marker_ok = self
372            .config
373            .root
374            .join(format!(".{}", self.config.app_name))
375            .is_dir();
376        if root_ok && marker_ok {
377            return Ok(None);
378        }
379        if !self
380            .all_states()?
381            .iter()
382            .any(|(_, s)| s.disk.hash.is_some())
383        {
384            return Ok(None);
385        }
386        Ok(Some(if root_ok {
387            PauseReason::MarkerMissing
388        } else {
389            PauseReason::RootMissing
390        }))
391    }
392
393    #[cfg(any(test, feature = "test-util"))]
394    fn fault_check(&mut self) -> Result<()> {
395        use crate::testing::FaultPoint;
396        if self.fault == Some(FaultPoint::AfterCommitBeforeWrite) {
397            self.fault = None;
398            return Err(Error::InjectedFault);
399        }
400        Ok(())
401    }
402
403    #[cfg(not(any(test, feature = "test-util")))]
404    fn fault_check(&mut self) -> Result<()> {
405        Ok(())
406    }
407
408    /// Called after each file written or removed. A crash loses what is still pending.
409    #[cfg(any(test, feature = "test-util"))]
410    fn crash_check(&mut self) -> Result<()> {
411        use crate::testing::FaultPoint;
412        match self.fault {
413            Some(FaultPoint::CrashAfterWrites(n)) if n > 1 => {
414                self.fault = Some(FaultPoint::CrashAfterWrites(n - 1));
415            }
416            Some(FaultPoint::CrashAfterWrites(_)) => {
417                self.fault = None;
418                self.pending.clear();
419                self.pending_versions = false;
420                return Err(Error::InjectedFault);
421            }
422            _ => {}
423        }
424        Ok(())
425    }
426
427    #[cfg(not(any(test, feature = "test-util")))]
428    fn crash_check(&mut self) -> Result<()> {
429        Ok(())
430    }
431
432    #[cfg(any(test, feature = "test-util"))]
433    fn run_reconcile_hook(&mut self) {
434        if let Some(hook) = self.reconcile_hook.take() {
435            hook();
436        }
437    }
438
439    #[cfg(not(any(test, feature = "test-util")))]
440    fn run_reconcile_hook(&mut self) {}
441
442    /// Record every unrecorded edit under the root and finish pending writes.
443    pub fn scan(&mut self) -> Result<ScanOutcome> {
444        if let Some(reason) = self.pause_reason()? {
445            return Ok(ScanOutcome::Paused(reason));
446        }
447        let result = self.scan_root();
448        let report = self.finish(result)?;
449        self.collect_staging_quietly();
450        Ok(ScanOutcome::Scanned(report))
451    }
452
453    fn scan_root(&mut self) -> Result<Report> {
454        let filter = self.filter()?;
455        let mut report = Report::default();
456        let mut rels = BTreeSet::new();
457        if self.config.root.is_dir() {
458            let root = self.config.root.clone();
459            let walker = walkdir::WalkDir::new(&root)
460                .follow_links(false)
461                .into_iter()
462                .filter_entry(|e| {
463                    e.depth() == 0
464                        || paths::rel_from_native(&root, e.path())
465                            .is_some_and(|rel| filter.allows(&rel, e.file_type().is_dir()))
466                });
467            for item in walker {
468                // One unreadable directory or Windows-locked file must not abort the
469                // whole scan: report it like any other per-path I/O error and keep
470                // walking, which `walkdir` does for the entry's siblings.
471                let item = match item {
472                    Ok(item) => item,
473                    Err(e) => {
474                        let path = e
475                            .path()
476                            .and_then(|p| paths::rel_from_native(&root, p))
477                            .unwrap_or_default();
478                        report.skipped.push(Skipped {
479                            path,
480                            reason: SkipReason::Io(e.to_string()),
481                        });
482                        continue;
483                    }
484                };
485                if item.depth() == 0 || item.file_type().is_dir() {
486                    continue;
487                }
488                let Some(rel) = paths::rel_from_native(&root, item.path()) else {
489                    continue;
490                };
491                if item.file_type().is_symlink() {
492                    report.skipped.push(Skipped {
493                        path: rel,
494                        reason: SkipReason::Symlink,
495                    });
496                    continue;
497                }
498                rels.insert(rel);
499            }
500        }
501        let states = self.all_states()?;
502        let mut index = StoreIndex::build(&states);
503        for (rel, _) in states {
504            rels.insert(rel);
505        }
506        for rel in rels {
507            let result = self.reconcile_path(&rel, &filter, None, &mut index, &mut report);
508            tolerate_io(result, &rel, &mut report)?;
509            self.run_reconcile_hook();
510        }
511        Ok(report)
512    }
513
514    /// Path states the peer with version vector `peer` has not merged.
515    pub fn delta_for(&self, peer: &VersionVector) -> Result<Vec<PathUpdate>> {
516        Ok(self
517            .store
518            .all()?
519            .into_iter()
520            .filter(|(_, s)| !peer.covers(&s.seen))
521            .map(|(path, s)| PathUpdate {
522                path,
523                versions: s.versions,
524                seen: s.seen,
525            })
526            .collect())
527    }
528
529    /// Join updates received from a peer, fetching content from `source`.
530    ///
531    /// Every update is joined before any file is written, so that the joined states go
532    /// to the store in one transaction and what the files then hold in another, rather
533    /// than several transactions per path.
534    pub fn apply(&mut self, updates: &[PathUpdate], source: &dyn ContentSource) -> Result<Report> {
535        if let Some(reason) = self.pause_reason()? {
536            return Err(Error::Paused(reason));
537        }
538        let result = self.apply_updates(updates, source);
539        self.finish(result)
540    }
541
542    fn apply_updates(
543        &mut self,
544        updates: &[PathUpdate],
545        source: &dyn ContentSource,
546    ) -> Result<Report> {
547        let filter = self.filter()?;
548        let mut report = Report::default();
549        let mut index = StoreIndex::build(&self.all_states()?);
550        let now = self.clock.now_ms();
551        let mut joined = BTreeSet::new();
552        for update in updates {
553            // A counter of 0 is not a dot any replica assigns (`next_entry` increments
554            // first), and it has no predecessor for `disk_seen` to pin a loser at.
555            let well_formed = paths::is_valid_rel(&update.path)
556                && !update.versions.is_empty()
557                && update
558                    .versions
559                    .iter()
560                    .all(|v| v.path == update.path && v.dot.counter > 0);
561            if !well_formed {
562                tracing::warn!(path = %update.path, "ignoring a malformed path update");
563                continue;
564            }
565            for version in &update.versions {
566                self.meta.hlc = self.meta.hlc.observe(version.hlc, now);
567            }
568            // Report a per-path I/O error and still integrate the update: the
569            // on-disk check in `settle` is what keeps this from overwriting an
570            // unrecorded local edit, and every delivered update must still be
571            // joined into the store for `commit_session` to be correct.
572            let reconciled =
573                self.reconcile_path(&update.path, &filter, Some(source), &mut index, &mut report);
574            tolerate_io(reconciled, &update.path, &mut report)?;
575            if self
576                .join(update.clone(), None, &mut index, &mut report)?
577                .is_some()
578            {
579                joined.insert(update.path.clone());
580            }
581        }
582        for rel in joined {
583            let Some(state) = self.get_state(&rel)? else {
584                continue;
585            };
586            let settled = self
587                .ensure_conflict_copies(
588                    &rel,
589                    &state,
590                    &filter,
591                    Some(source),
592                    &mut index,
593                    &mut report,
594                )
595                .and_then(|()| {
596                    self.settle(&rel, state, &filter, Some(source), &mut index, &mut report)
597                });
598            tolerate_io(settled, &rel, &mut report)?;
599        }
600        Ok(report)
601    }
602
603    /// Declare that every update of a session with `peer` has been applied.
604    pub fn commit_session(&mut self, peer: &VersionVector) -> Result<()> {
605        self.meta.vv.merge(peer);
606        if self.pending.is_empty() {
607            self.store.commit(&self.meta, &[])
608        } else {
609            self.flush()
610        }
611    }
612
613    /// Retry writes that were waiting for content.
614    pub fn fetch_missing(&mut self, source: &dyn ContentSource) -> Result<Report> {
615        if let Some(reason) = self.pause_reason()? {
616            return Err(Error::Paused(reason));
617        }
618        let result = self.fetch_missing_states(source);
619        self.finish(result)
620    }
621
622    fn fetch_missing_states(&mut self, source: &dyn ContentSource) -> Result<Report> {
623        let filter = self.filter()?;
624        let mut report = Report::default();
625        let states = self.all_states()?;
626        let mut index = StoreIndex::build(&states);
627        for (rel, _) in states {
628            let result = self.reconcile_path(&rel, &filter, Some(source), &mut index, &mut report);
629            tolerate_io(result, &rel, &mut report)?;
630        }
631        Ok(report)
632    }
633
634    /// Content this replica wants written and does not have: the winner of a path whose
635    /// file does not hold it yet, and each loser still waiting for its conflict copy.
636    ///
637    /// A session asks its peer for these whatever the peer sends: committing a session
638    /// takes in the peer's version vector even when some content never arrived, so no
639    /// later delta carries those paths again. Content the replica would not write anyway
640    /// — an ignored or unrepresentable path, a case collision, a file over the size limit
641    /// — is left out.
642    pub fn unavailable(&self) -> Result<Vec<ContentHash>> {
643        let filter = self.filter()?;
644        let states = self.all_states()?;
645        let index = StoreIndex::build(&states);
646        let max = self.config.max_file_size;
647        let mut wanted: BTreeSet<ContentHash> = BTreeSet::new();
648        for (rel, state) in &states {
649            let winner = state.winner();
650            if let Content::File { hash, .. } = winner.content
651                && state.disk.hash != Some(hash)
652                && self.skip_reason(rel, winner, &filter, &index)?.is_none()
653            {
654                wanted.insert(hash);
655            }
656            for loser in state
657                .versions
658                .iter()
659                .filter(|v| v.dot != winner.dot && merge::needs_copy(v, winner))
660            {
661                let Content::File { hash, len } = loser.content else {
662                    continue;
663                };
664                let copy_rel = merge::conflict_path(rel, &loser.dot);
665                if state.disk.hash != Some(hash)
666                    && len <= max
667                    && filter.allows(&copy_rel, false)
668                    && paths::representable(&copy_rel)
669                    && self.get_state(&copy_rel)?.is_none()
670                {
671                    wanted.insert(hash);
672                }
673            }
674        }
675        let mut missing = Vec::new();
676        for hash in wanted {
677            if self.content_path_with(&hash, &index)?.is_none() {
678                missing.push(hash);
679            }
680        }
681        Ok(missing)
682    }
683
684    /// Remove staged content nothing needs any more, returning how many files went.
685    ///
686    /// Staged content is kept while some stored version refers to it and no workspace
687    /// file holds it: the replica may still write it, or be asked for it. Placing content
688    /// moves it out of staging, so what this removes is content staged and never placed —
689    /// a version superseded first, a second copy from another session, inline content of
690    /// a page that never came. A workspace file counts as holding the content only when
691    /// its recorded stamp still matches and is not racy; nothing is re-hashed here.
692    ///
693    /// Files younger than [`STAGING_GRACE`] are kept whatever they hold: a session stages
694    /// inline content before the page that refers to it arrives, and runs alongside this.
695    pub fn collect_staging(&self) -> Result<usize> {
696        let staged = staging::finished(&self.config.staging_dir)?;
697        if staged.is_empty() {
698            return Ok(0);
699        }
700        let states = self.all_states()?;
701        let referenced: BTreeSet<ContentHash> = states
702            .iter()
703            .flat_map(|(_, state)| &state.versions)
704            .filter_map(|v| v.content.hash())
705            .collect();
706        let index = StoreIndex::build(&states);
707        let now = SystemTime::now();
708        let mut removed = 0;
709        for (hash, path, meta) in staged {
710            let young = meta
711                .modified()
712                .ok()
713                .and_then(|m| now.duration_since(m).ok())
714                .is_none_or(|age| age < STAGING_GRACE);
715            if young {
716                continue;
717            }
718            if referenced.contains(&hash) && !self.held_in_workspace(&hash, &index)? {
719                continue;
720            }
721            match fs::remove_file(&path) {
722                Ok(()) => removed += 1,
723                Err(e) if e.kind() == ErrorKind::NotFound => {}
724                Err(e) => {
725                    tracing::debug!(path = %path.display(), "could not remove staged content: {e}")
726                }
727            }
728        }
729        Ok(removed)
730    }
731
732    /// [`Self::collect_staging`], logging rather than returning a failure: collecting is
733    /// housekeeping, and must not fail the open or scan it follows.
734    fn collect_staging_quietly(&self) {
735        match self.collect_staging() {
736            Ok(0) => {}
737            Ok(n) => tracing::debug!(removed = n, "collected staged content"),
738            Err(e) => tracing::warn!("could not collect staged content: {e}"),
739        }
740    }
741
742    /// Whether a workspace file holds `hash`, judged by its recorded stamp alone.
743    fn held_in_workspace(&self, hash: &ContentHash, index: &StoreIndex) -> Result<bool> {
744        for rel in index.paths_with(hash) {
745            let Some(state) = self.get_state(rel)? else {
746                continue;
747            };
748            let disk = &state.disk;
749            if !is_racy(disk)
750                && fs::metadata(paths::to_native(&self.config.root, rel))
751                    .is_ok_and(|meta| stamp_of(&meta) == (disk.mtime_ns, disk.len))
752            {
753                return Ok(true);
754            }
755        }
756        Ok(false)
757    }
758
759    /// A [`Stager`] over this replica's staging directory and size limit.
760    pub fn stager(&self) -> Stager {
761        Stager::new(self.config.staging_dir.clone(), self.config.max_file_size)
762    }
763
764    /// A file holding the content `hash`, and its length: a synced file on disk, else
765    /// finished staged content. A file whose length and mtime still match what was
766    /// recorded (and are not racy) is trusted; any other is re-hashed first.
767    pub fn content_path(&self, hash: &ContentHash) -> Result<Option<(PathBuf, u64)>> {
768        self.content_path_with(hash, &StoreIndex::build(&self.all_states()?))
769    }
770
771    /// [`Self::content_path`] for each of `hashes`, building the store index once for
772    /// all of them rather than once per hash. A hash no file or staged content holds is
773    /// left out of the map.
774    pub fn content_paths(
775        &self,
776        hashes: &[ContentHash],
777    ) -> Result<HashMap<ContentHash, (PathBuf, u64)>> {
778        let index = StoreIndex::build(&self.all_states()?);
779        let mut found = HashMap::new();
780        for hash in hashes {
781            if found.contains_key(hash) {
782                continue;
783            }
784            if let Some(hit) = self.content_path_with(hash, &index)? {
785                found.insert(*hash, hit);
786            }
787        }
788        Ok(found)
789    }
790
791    /// [`Self::content_path`] against an index the caller already holds.
792    fn content_path_with(
793        &self,
794        hash: &ContentHash,
795        index: &StoreIndex,
796    ) -> Result<Option<(PathBuf, u64)>> {
797        for rel in index.paths_with(hash) {
798            let Some(state) = self.get_state(rel)? else {
799                continue;
800            };
801            let path = paths::to_native(&self.config.root, rel);
802            let Ok(meta) = fs::metadata(&path) else {
803                continue;
804            };
805            let (mtime_ns, len) = stamp_of(&meta);
806            let disk = &state.disk;
807            let trusted = disk.hash.as_ref() == Some(hash)
808                && (mtime_ns, len) == (disk.mtime_ns, disk.len)
809                && !is_racy(disk);
810            if trusted || ContentHash::of_file(&path).is_ok_and(|h| h == *hash) {
811                return Ok(Some((path, len)));
812            }
813        }
814        if let Some(path) = self.stager().staged(hash)
815            && let Ok(meta) = fs::metadata(&path)
816        {
817            return Ok(Some((path, meta.len())));
818        }
819        Ok(None)
820    }
821
822    fn next_entry(&mut self, rel: &str, content: Content, context: VersionVector) -> Entry {
823        self.meta.counter += 1;
824        self.meta.hlc = self.meta.hlc.tick(self.clock.now_ms());
825        let dot = Dot {
826            replica: self.meta.replica_id,
827            counter: self.meta.counter,
828        };
829        self.meta.vv.add_dot(&dot);
830        Entry {
831            path: rel.to_owned(),
832            content,
833            hlc: self.meta.hlc,
834            dot,
835            context,
836            author: self.config.device_id,
837        }
838    }
839
840    /// Compare one path's file with its state: record an edit, or settle the state.
841    /// A file holding a version the store already knows is settled, not recorded: a
842    /// watcher-driven scan of content a live session just materialized must not re-record it
843    /// as a local edit (issue #157).
844    fn reconcile_path(
845        &mut self,
846        rel: &str,
847        filter: &SyncFilter,
848        source: Option<&dyn ContentSource>,
849        index: &mut StoreIndex,
850        report: &mut Report,
851    ) -> Result<()> {
852        if !filter.allows(rel, false) || !paths::representable(rel) {
853            return Ok(());
854        }
855        let abs = paths::to_native(&self.config.root, rel);
856        let state = self.get_state(rel)?;
857        let disk = state.as_ref().map(|s| s.disk.clone()).unwrap_or_default();
858        // On a case-insensitive filesystem, `a.txt` would find the file of `A.txt`.
859        if disk.hash.is_none() && index.case_twin(rel).is_some() {
860            if let Some(state) = state {
861                self.settle(rel, state, filter, source, index, report)?;
862            }
863            return Ok(());
864        }
865        let fs_meta = match fs::symlink_metadata(&abs) {
866            Ok(m) => Some(m),
867            Err(e) if e.kind() == ErrorKind::NotFound => None,
868            Err(e) => return Err(e.into()),
869        };
870        let mut stamp = (0, 0);
871        let file_hash = match &fs_meta {
872            Some(m) if m.file_type().is_symlink() => {
873                report.skipped.push(Skipped {
874                    path: rel.to_owned(),
875                    reason: SkipReason::Symlink,
876                });
877                return Ok(());
878            }
879            Some(m) if m.is_file() => {
880                stamp = stamp_of(m);
881                if disk.hash.is_some() && stamp == (disk.mtime_ns, disk.len) && !is_racy(&disk) {
882                    disk.hash
883                } else if m.len() > self.config.max_file_size {
884                    report.skipped.push(Skipped {
885                        path: rel.to_owned(),
886                        reason: SkipReason::TooLarge {
887                            len: m.len(),
888                            max: self.config.max_file_size,
889                        },
890                    });
891                    return Ok(());
892                } else {
893                    Some(ContentHash::of_file(&abs)?)
894                }
895            }
896            _ => None,
897        };
898
899        if file_hash == disk.hash {
900            if let Some(mut state) = state {
901                if file_hash.is_some() && (stamp != (disk.mtime_ns, disk.len) || is_racy(&disk)) {
902                    state.disk.mtime_ns = stamp.0;
903                    state.disk.len = stamp.1;
904                    state.disk.checked_ns = now_ns();
905                    self.commit_state(rel, &state, index)?;
906                }
907                self.ensure_conflict_copies(rel, &state, filter, source, index, report)?;
908                self.settle(rel, state, filter, source, index, report)?;
909            }
910            return Ok(());
911        }
912
913        // A watcher-driven scan can fire after a live session has already committed a peer's
914        // version to the store and written its bytes to disk. The file then holds content the
915        // store already knows, but `disk.hash` still names the file the commit replaced, so
916        // `file_hash != disk.hash`. Recording it here would invent a fresh own dot, sibling to
917        // the version just received: a spurious conflict copy, and a delete context polluted
918        // enough to stop a later delete from propagating (issue #157). Content the store
919        // already knows is a materialization to settle, not a local edit to record.
920        //
921        // Matched against the *winner* specifically, not any sibling (issue #161). A path in
922        // conflict keeps every loser's bytes alive too — as a conflict copy next to it — so a
923        // local write can land on a loser's content without having materialized anything: a
924        // deliberate edit that merely coincides with a version already present elsewhere in
925        // the conflict set, not a re-appearance of what this scan's own disk state already
926        // reflects. The winner is the one version this branch's premise actually describes
927        // (disk already *is* this content, only the bookkeeping is stale), and it is also
928        // `settle`'s own criterion for "nothing to do" once `disk.hash` is refreshed below —
929        // matching any sibling let a write that changes which content wins slip through
930        // silently, destroying the winner's only copy (the file) with nothing recording that
931        // anything superseded it, and no conflict copy to fall back on (conflict copies exist
932        // for losers, never for the winner).
933        //
934        // A missing file whose winner is a tombstone is the same case: the delete was
935        // carried out, and a crash came before it was recorded.
936        if state
937            .as_ref()
938            .is_some_and(|s| s.winner().content.hash() == file_hash)
939        {
940            if let Some(mut state) = state {
941                // The on-disk file now holds `file_hash`, but the stored `disk` still
942                // describes the file the commit replaced; refresh it so `settle` sees the
943                // disk as already matching the known version instead of re-materializing it.
944                if state.disk.hash != file_hash
945                    || stamp != (disk.mtime_ns, disk.len)
946                    || is_racy(&disk)
947                {
948                    state.disk.hash = file_hash;
949                    state.disk.mtime_ns = stamp.0;
950                    state.disk.len = stamp.1;
951                    state.disk.checked_ns = now_ns();
952                    self.commit_state(rel, &state, index)?;
953                }
954                self.ensure_conflict_copies(rel, &state, filter, source, index, report)?;
955                self.settle(rel, state, filter, source, index, report)?;
956            }
957            return Ok(());
958        }
959
960        let content = match file_hash {
961            Some(hash) => Content::File { hash, len: stamp.1 },
962            None => Content::Tombstone,
963        };
964        if state.is_none() && content.is_tombstone() {
965            return Ok(());
966        }
967        // A root that disappears *during* a scan looks exactly like a bulk delete: the
968        // entry guard already passed, so every remaining path would be tombstoned and
969        // the next push would delete them everywhere. Re-check before recording a
970        // delete for a file this replica had materialized. `Error::Paused` is not
971        // `Error::Io`, so `tolerate_io` propagates it and the scan stops here.
972        if content.is_tombstone()
973            && disk.hash.is_some()
974            && let Some(reason) = self.pause_reason()?
975        {
976            return Err(Error::Paused(reason));
977        }
978        let entry = self.next_entry(rel, content, disk.seen.clone());
979        report.recorded.push(entry.clone());
980        let mut seen = entry.context.clone();
981        seen.add_dot(&entry.dot);
982        let new_disk = DiskState {
983            hash: file_hash,
984            seen: seen.clone(),
985            mtime_ns: stamp.0,
986            len: stamp.1,
987            checked_ns: now_ns(),
988        };
989        let update = PathUpdate {
990            path: rel.to_owned(),
991            versions: vec![entry],
992            seen,
993        };
994        self.integrate(update, Some(new_disk), filter, source, index, report)
995    }
996
997    /// Join `update` into the stored state and commit, then settle the file.
998    /// `local_disk` is set when the update is a local write already on disk.
999    fn integrate(
1000        &mut self,
1001        update: PathUpdate,
1002        local_disk: Option<DiskState>,
1003        filter: &SyncFilter,
1004        source: Option<&dyn ContentSource>,
1005        index: &mut StoreIndex,
1006        report: &mut Report,
1007    ) -> Result<()> {
1008        let rel = update.path.clone();
1009        let Some(state) = self.join(update, local_disk, index, report)? else {
1010            return Ok(());
1011        };
1012        // Copies first: settling may overwrite the loser's bytes on disk.
1013        self.ensure_conflict_copies(&rel, &state, filter, source, index, report)?;
1014        self.settle(&rel, state, filter, source, index, report)
1015    }
1016
1017    /// Join `update` into the stored state and commit it, leaving the file as it is.
1018    /// The joined state, or `None` when the update brought nothing new.
1019    fn join(
1020        &mut self,
1021        update: PathUpdate,
1022        local_disk: Option<DiskState>,
1023        index: &mut StoreIndex,
1024        report: &mut Report,
1025    ) -> Result<Option<PathState>> {
1026        let rel = update.path;
1027        let old = self.get_state(&rel)?;
1028        let joined = merge::join(
1029            old.as_ref().map(|s| (s.versions.as_slice(), &s.seen)),
1030            &update.versions,
1031            &update.seen,
1032        );
1033        let Some((versions, seen)) = joined else {
1034            return Ok(None);
1035        };
1036        let disk = match local_disk {
1037            Some(d) => d,
1038            None => old.map(|s| s.disk).unwrap_or_default(),
1039        };
1040        let state = PathState {
1041            versions,
1042            seen,
1043            disk,
1044        };
1045        self.commit_state(&rel, &state, index)?;
1046        self.pending_versions = true;
1047        report.changed += 1;
1048        Ok(Some(state))
1049    }
1050
1051    /// Write a conflict copy for every loser that needs one and has never had one.
1052    fn ensure_conflict_copies(
1053        &mut self,
1054        rel: &str,
1055        state: &PathState,
1056        filter: &SyncFilter,
1057        source: Option<&dyn ContentSource>,
1058        index: &mut StoreIndex,
1059        report: &mut Report,
1060    ) -> Result<()> {
1061        if state.versions.len() < 2 {
1062            return Ok(());
1063        }
1064        let winner = state.winner().clone();
1065        let losers: Vec<Entry> = state
1066            .versions
1067            .iter()
1068            .filter(|v| v.dot != winner.dot && merge::needs_copy(v, &winner))
1069            .cloned()
1070            .collect();
1071        for loser in losers {
1072            let Content::File { hash, .. } = loser.content else {
1073                continue;
1074            };
1075            let copy_rel = merge::conflict_path(rel, &loser.dot);
1076            if !filter.allows(&copy_rel, false) {
1077                report.skipped.push(Skipped {
1078                    path: copy_rel,
1079                    reason: SkipReason::Ignored,
1080                });
1081                continue;
1082            }
1083            if !paths::representable(&copy_rel) {
1084                report.skipped.push(Skipped {
1085                    path: copy_rel,
1086                    reason: SkipReason::Unrepresentable,
1087                });
1088                continue;
1089            }
1090            if self.get_state(&copy_rel)?.is_some() {
1091                continue;
1092            }
1093            let copy_abs = paths::to_native(&self.config.root, &copy_rel);
1094            // Only the loser's own bytes count as the copy already being there. Anything
1095            // else at the copy path is an unrelated file, and recording it as the copy
1096            // (which the reconcile below does) would lose the loser silently.
1097            let holds_the_loser = match fs::symlink_metadata(&copy_abs) {
1098                Ok(m) if m.is_file() && ContentHash::of_file(&copy_abs)? == hash => true,
1099                // A regular file with different bytes, a directory, or a symlink: not
1100                // the loser's bytes, and not something to write over. The loser stays
1101                // out of `disk.seen`, so it is not lost.
1102                Ok(_) => {
1103                    report.skipped.push(Skipped {
1104                        path: copy_rel,
1105                        reason: SkipReason::Occupied,
1106                    });
1107                    continue;
1108                }
1109                Err(e) if e.kind() == ErrorKind::NotFound => false,
1110                Err(e) => return Err(e.into()),
1111            };
1112            if !holds_the_loser {
1113                // The loser may be exactly what is on disk at `rel` right now. A read
1114                // error here must propagate rather than be swallowed as unavailable
1115                // content: `settle` would otherwise overwrite the only local copy of
1116                // the loser's bytes.
1117                //
1118                // The file on disk is copied whatever its size: the limit may have
1119                // been lowered since it was recorded, and the copy is what keeps the
1120                // loser from being overwritten.
1121                let staged = if state.disk.hash == Some(hash) {
1122                    let unbounded = Stager::new(self.config.staging_dir.clone(), u64::MAX);
1123                    let from = paths::to_native(&self.config.root, rel);
1124                    Self::stage_with(&unbounded, &from, &hash)?
1125                } else {
1126                    let from = match self.content_path_with(&hash, index)? {
1127                        Some((path, _)) => Some(path),
1128                        None => source.and_then(|s| s.locate(&hash)),
1129                    };
1130                    match from {
1131                        Some(from) => self.stage_from(&from, &hash)?,
1132                        None => None,
1133                    }
1134                };
1135                let Some(staged) = staged else {
1136                    report.skipped.push(Skipped {
1137                        path: copy_rel,
1138                        reason: SkipReason::ContentUnavailable,
1139                    });
1140                    continue;
1141                };
1142                self.flush_versions()?;
1143                place(&staged, &copy_abs)?;
1144                report.conflicts.push(Conflict {
1145                    path: rel.to_owned(),
1146                    copy_path: copy_rel.clone(),
1147                });
1148            }
1149            // Record the copy (or a file someone already put there) as a local write.
1150            self.reconcile_path(&copy_rel, filter, source, index, report)?;
1151        }
1152        Ok(())
1153    }
1154
1155    /// Make the file on disk hold the winner, then record what it reflects.
1156    fn settle(
1157        &mut self,
1158        rel: &str,
1159        mut state: PathState,
1160        filter: &SyncFilter,
1161        source: Option<&dyn ContentSource>,
1162        index: &mut StoreIndex,
1163        report: &mut Report,
1164    ) -> Result<()> {
1165        let winner = state.winner().clone();
1166        if winner.content.hash() == state.disk.hash {
1167            let seen = self.disk_seen(&state, filter)?;
1168            if seen != state.disk.seen {
1169                state.disk.seen = seen;
1170                self.commit_state(rel, &state, index)?;
1171            }
1172            return Ok(());
1173        }
1174        if let Some(reason) = self.skip_reason(rel, &winner, filter, index)? {
1175            report.skipped.push(Skipped {
1176                path: rel.to_owned(),
1177                reason,
1178            });
1179            return Ok(());
1180        }
1181        if self.occupied(rel, &state.disk)? {
1182            report.skipped.push(Skipped {
1183                path: rel.to_owned(),
1184                reason: SkipReason::Occupied,
1185            });
1186            return Ok(());
1187        }
1188        let abs = paths::to_native(&self.config.root, rel);
1189        match winner.content {
1190            Content::Tombstone => {
1191                self.flush_versions()?;
1192                self.fault_check()?;
1193                match fs::remove_file(&abs) {
1194                    Ok(()) => {}
1195                    Err(e) if e.kind() == ErrorKind::NotFound => {}
1196                    Err(e) => return Err(e.into()),
1197                }
1198                self.crash_check()?;
1199                state.disk = DiskState::default();
1200            }
1201            Content::File { hash, .. } => {
1202                let Some(staged) = self.stage(&hash, source, index)? else {
1203                    report.skipped.push(Skipped {
1204                        path: rel.to_owned(),
1205                        reason: SkipReason::ContentUnavailable,
1206                    });
1207                    return Ok(());
1208                };
1209                self.flush_versions()?;
1210                self.fault_check()?;
1211                place(&staged, &abs)?;
1212                self.crash_check()?;
1213                let (mtime_ns, len) = stamp_of(&fs::metadata(&abs)?);
1214                state.disk = DiskState {
1215                    hash: Some(hash),
1216                    seen: VersionVector::new(),
1217                    mtime_ns,
1218                    len,
1219                    checked_ns: now_ns(),
1220                };
1221            }
1222        }
1223        state.disk.seen = self.disk_seen(&state, filter)?;
1224        self.commit_state(rel, &state, index)
1225    }
1226
1227    /// Commit one path state and keep `index` in step with it. The state is pending
1228    /// until [`Self::flush`], which this calls once enough states are pending.
1229    fn commit_state(&mut self, rel: &str, state: &PathState, index: &mut StoreIndex) -> Result<()> {
1230        self.pending.insert(rel.to_owned(), state.clone());
1231        index.record(rel, state.disk.hash);
1232        if self.pending.len() >= MAX_PENDING {
1233            self.flush()?;
1234        }
1235        Ok(())
1236    }
1237
1238    /// Versions the file on disk reflects once the winner is written: everything
1239    /// merged, except losers still waiting for a conflict copy — a local edit must
1240    /// not supersede a version whose bytes were never preserved. A loser whose copy
1241    /// path the filter rejects, or that the local OS cannot represent, is never
1242    /// excluded this way: no copy will ever be attempted for it, so pinning it here
1243    /// would keep it out of `seen` forever.
1244    fn disk_seen(&self, state: &PathState, filter: &SyncFilter) -> Result<VersionVector> {
1245        let winner = state.winner();
1246        let mut seen = state.seen.clone();
1247        for loser in state
1248            .versions
1249            .iter()
1250            .filter(|v| v.dot != winner.dot && merge::needs_copy(v, winner))
1251        {
1252            let copy_rel = merge::conflict_path(&loser.path, &loser.dot);
1253            if !filter.allows(&copy_rel, false) || !paths::representable(&copy_rel) {
1254                continue;
1255            }
1256            if self.get_state(&copy_rel)?.is_none() {
1257                let slot = seen.0.entry(loser.dot.replica).or_insert(0);
1258                *slot = (*slot).min(loser.dot.counter.saturating_sub(1));
1259            }
1260        }
1261        Ok(seen)
1262    }
1263
1264    fn skip_reason(
1265        &self,
1266        rel: &str,
1267        winner: &Entry,
1268        filter: &SyncFilter,
1269        index: &StoreIndex,
1270    ) -> Result<Option<SkipReason>> {
1271        if !filter.allows(rel, false) {
1272            return Ok(Some(SkipReason::Ignored));
1273        }
1274        if !paths::representable(rel) {
1275            return Ok(Some(SkipReason::Unrepresentable));
1276        }
1277        if let Content::File { len, .. } = winner.content
1278            && len > self.config.max_file_size
1279        {
1280            return Ok(Some(SkipReason::TooLarge {
1281                len,
1282                max: self.config.max_file_size,
1283            }));
1284        }
1285        if !winner.content.is_tombstone()
1286            && let Some(other) = index.case_twin(rel)
1287        {
1288            return Ok(Some(SkipReason::CaseCollision { other }));
1289        }
1290        Ok(None)
1291    }
1292
1293    /// Whether the path on disk at `rel` holds something this replica has not
1294    /// recorded: a symlink, a directory, a file over the size cap, or a file whose
1295    /// content differs from `disk.hash`. A missing file, or a file matching
1296    /// `disk.hash`, is not occupied.
1297    fn occupied(&self, rel: &str, disk: &DiskState) -> Result<bool> {
1298        let abs = paths::to_native(&self.config.root, rel);
1299        let meta = match fs::symlink_metadata(&abs) {
1300            Ok(m) => m,
1301            Err(e) if e.kind() == ErrorKind::NotFound => return Ok(false),
1302            Err(e) => return Err(e.into()),
1303        };
1304        if !meta.is_file() {
1305            // A symlink or a directory.
1306            return Ok(true);
1307        }
1308        if meta.len() > self.config.max_file_size {
1309            return Ok(true);
1310        }
1311        let stamp = stamp_of(&meta);
1312        let hash = if disk.hash.is_some() && stamp == (disk.mtime_ns, disk.len) && !is_racy(disk) {
1313            disk.hash
1314        } else {
1315            Some(ContentHash::of_file(&abs)?)
1316        };
1317        Ok(hash != disk.hash)
1318    }
1319
1320    /// A verified copy of `hash` in the staging directory, if one can be had.
1321    fn stage(
1322        &self,
1323        hash: &ContentHash,
1324        source: Option<&dyn ContentSource>,
1325        index: &StoreIndex,
1326    ) -> Result<Option<PathBuf>> {
1327        // Anything staged under its hash was verified when it was finished.
1328        if let Some(staged) = self.stager().staged(hash) {
1329            return Ok(Some(staged));
1330        }
1331        let from = match self.content_path_with(hash, index)? {
1332            Some((path, _)) => path,
1333            None => match source.and_then(|s| s.locate(hash)) {
1334                Some(path) => path,
1335                None => return Ok(None),
1336            },
1337        };
1338        self.stage_from(&from, hash)
1339    }
1340
1341    /// Copy the file `from` into the staging directory as the content `hash`, within
1342    /// the size limit. Content that is over the limit or does not hash to `hash` is
1343    /// logged and treated as unavailable; a failure to read `from` or write the copy
1344    /// is an error.
1345    fn stage_from(&self, from: &Path, hash: &ContentHash) -> Result<Option<PathBuf>> {
1346        Self::stage_with(&self.stager(), from, hash)
1347    }
1348
1349    /// [`Self::stage_from`] through the given stager.
1350    fn stage_with(stager: &Stager, from: &Path, hash: &ContentHash) -> Result<Option<PathBuf>> {
1351        let mut file = fs::File::open(from)?;
1352        let len = file.metadata()?.len();
1353        let mut writer = match stager.writer(*hash, len) {
1354            Ok(writer) => writer,
1355            Err(Error::TooLarge { len, max }) => {
1356                tracing::warn!(%hash, len, max, "content is over the size limit");
1357                return Ok(None);
1358            }
1359            Err(e) => return Err(e),
1360        };
1361        // A file that grows after its length was read overruns the writer mid-copy.
1362        if let Err(e) = std::io::copy(&mut file, &mut writer) {
1363            return match e.get_ref().and_then(|i| i.downcast_ref::<Error>()) {
1364                Some(Error::TooLarge { .. } | Error::ContentMismatch { .. }) => {
1365                    tracing::warn!(%hash, "content changed while it was copied");
1366                    Ok(None)
1367                }
1368                _ => Err(e.into()),
1369            };
1370        }
1371        match writer.finish() {
1372            Ok(path) => Ok(Some(path)),
1373            Err(Error::ContentMismatch { .. }) => {
1374                tracing::warn!(%hash, "content did not match its hash");
1375                Ok(None)
1376            }
1377            Err(e) => Err(e),
1378        }
1379    }
1380}
1381
1382impl ContentSource for Replica {
1383    fn locate(&self, hash: &ContentHash) -> Option<PathBuf> {
1384        self.content_path(hash).ok().flatten().map(|(path, _)| path)
1385    }
1386}
1387
1388#[cfg(any(test, feature = "test-util"))]
1389impl Replica {
1390    /// Open with a fixed replica id (used only when the store is created).
1391    pub fn open_with_replica_id(
1392        config: ReplicaConfig,
1393        clock: Arc<dyn Clock>,
1394        id: ReplicaId,
1395    ) -> Result<Self> {
1396        Self::open_inner(config, clock, Some(id))
1397    }
1398
1399    /// Fire `point` the next time it is reached, then clear it.
1400    pub fn inject_fault(&mut self, point: crate::testing::FaultPoint) {
1401        self.fault = Some(point);
1402    }
1403
1404    /// Run `hook` once after the next path a scan reconciles, then clear it.
1405    pub fn inject_after_reconcile(&mut self, hook: Box<dyn FnOnce() + Send>) {
1406        self.reconcile_hook = Some(hook);
1407    }
1408
1409    /// Everything that must agree across converged replicas, without timestamps of
1410    /// the local filesystem.
1411    pub fn logical_dump(&self) -> Result<serde_json::Value> {
1412        let mut path_states = serde_json::Map::new();
1413        for (rel, s) in self.all_states()? {
1414            path_states.insert(
1415                rel,
1416                serde_json::json!({
1417                    "versions": s.versions,
1418                    "seen": s.seen,
1419                    "disk_hash": s.disk.hash,
1420                    "disk_seen": s.disk.seen,
1421                }),
1422            );
1423        }
1424        let mut files = serde_json::Map::new();
1425        if self.config.root.is_dir() {
1426            for item in walkdir::WalkDir::new(&self.config.root).sort_by_file_name() {
1427                let item = item.map_err(|e| Error::Io(std::io::Error::other(e)))?;
1428                if !item.file_type().is_file() {
1429                    continue;
1430                }
1431                if let Some(rel) = paths::rel_from_native(&self.config.root, item.path()) {
1432                    files.insert(rel, serde_json::json!(ContentHash::of_file(item.path())?));
1433                }
1434            }
1435        }
1436        Ok(serde_json::json!({
1437            "replica_id": self.meta.replica_id,
1438            "vv": self.meta.vv,
1439            "paths": path_states,
1440            "files": files,
1441        }))
1442    }
1443}
1444
1445#[cfg(test)]
1446mod tests {
1447    use super::*;
1448
1449    fn hash(body: &str) -> ContentHash {
1450        ContentHash::of_bytes(body.as_bytes())
1451    }
1452
1453    #[test]
1454    fn the_index_follows_a_path_from_one_hash_to_another() {
1455        let mut index = StoreIndex::default();
1456        index.record("a.txt", Some(hash("one")));
1457        index.record("b.txt", Some(hash("one")));
1458        assert_eq!(
1459            index.paths_with(&hash("one")).collect::<Vec<_>>(),
1460            vec!["a.txt", "b.txt"]
1461        );
1462
1463        index.record("a.txt", Some(hash("two")));
1464        assert_eq!(
1465            index.paths_with(&hash("one")).collect::<Vec<_>>(),
1466            vec!["b.txt"],
1467            "the stale hash no longer points at a.txt"
1468        );
1469        assert_eq!(
1470            index.paths_with(&hash("two")).collect::<Vec<_>>(),
1471            vec!["a.txt"]
1472        );
1473
1474        // A deleted file has no bytes on disk any more.
1475        index.record("a.txt", None);
1476        assert_eq!(index.paths_with(&hash("two")).count(), 0);
1477    }
1478
1479    #[test]
1480    fn the_index_finds_a_materialized_case_twin() {
1481        let mut index = StoreIndex::default();
1482        index.record("A.txt", Some(hash("upper")));
1483        index.record("a.txt", None);
1484        let twin = index.case_twin("a.txt");
1485        if paths::CASE_INSENSITIVE_FS {
1486            assert_eq!(twin.as_deref(), Some("A.txt"));
1487            assert_eq!(index.case_twin("A.txt"), None, "a.txt is not on disk");
1488            assert_eq!(index.case_twin("other.txt"), None);
1489        } else {
1490            assert_eq!(twin, None, "no case twins on a case-sensitive filesystem");
1491        }
1492    }
1493}