kernoid_sync/
lib.rs

1// SPDX-License-Identifier: MIT
2//! kernoid-sync: a folder kept in step with a file store of kernoid (RFC 0005) – the logic every one of kernoid's apps
3//! shares, whatever shows the files (a folder, the system's file manager through a native shell).
4//!
5//! One round ([`Folder::sync`]): the store's changes since the state last read (`FileNode/changes`; the whole store
6//! when there is no state or it is too old), the folder scanned (size and modification time tell a change), both
7//! compared with what they were when they last agreed ([`plan`]), and the actions carried out. A change on both sides
8//! keeps both: the server's version takes the name, the local one becomes a copy beside it. The local state is a
9//! SQLite database outside the folder ([`state`]).
10//!
11//! OPEN: moves as moves (today a deletion and a creation), transfers resumed after an interruption (RFC 9404),
12//! content hashes to tell a change that keeps size and time, vaults (RFC 0006, 6: their names are encrypted), the
13//! file systems' own name rules (Windows).
14
15pub mod plan;
16pub mod state;
17
18use std::{
19    collections::BTreeMap,
20    fmt,
21    path::{Path, PathBuf},
22    time::UNIX_EPOCH,
23};
24
25use kernoid_client::{
26    Client, FILENODE,
27    files::{self, FileNode},
28};
29use serde_json::{Value, json};
30
31use plan::{Action, Base, Local, RelPath, Remote};
32use state::{Node, State};
33
34/// Changes asked for at once.
35const MAX_CHANGES: u64 = 500;
36/// The suffix of a download not finished yet; such files are not part of the folder.
37const PARTIAL: &str = ".kernoid-part";
38
39#[derive(Debug)]
40pub enum Error {
41    /// Talking to kernoid.
42    Client(kernoid_client::Error),
43    /// The local state.
44    State(String),
45    /// The folder.
46    Io(std::io::Error),
47}
48
49impl fmt::Display for Error {
50    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
51        match self {
52            Self::Client(error) => write!(f, "{error}"),
53            Self::State(error) => write!(f, "the local state: {error}"),
54            Self::Io(error) => write!(f, "{error}"),
55        }
56    }
57}
58
59impl std::error::Error for Error {}
60
61impl From<kernoid_client::Error> for Error {
62    fn from(error: kernoid_client::Error) -> Self {
63        Self::Client(error)
64    }
65}
66
67impl From<std::io::Error> for Error {
68    fn from(error: std::io::Error) -> Self {
69        Self::Io(error)
70    }
71}
72
73pub type Result<T> = std::result::Result<T, Error>;
74
75/// What a round did.
76#[derive(Debug, Clone, Default, PartialEq, Eq)]
77pub struct Report {
78    pub downloaded: usize,
79    pub uploaded: usize,
80    pub deleted_here: usize,
81    pub deleted_there: usize,
82    pub directories: usize,
83    /// Local files kept as copies beside the server's version.
84    pub conflicts: Vec<RelPath>,
85    /// Actions that failed, with why; the next round tries them again.
86    pub failed: Vec<(RelPath, String)>,
87    /// Nodes of the store left out: names a folder cannot hold.
88    pub skipped: Vec<String>,
89}
90
91/// A folder kept in step with one store (an account's own files, a drive, or files shared with the user).
92#[derive(Debug)]
93pub struct Folder {
94    root: PathBuf,
95    store: String,
96    state: State,
97}
98
99impl Folder {
100    /// The folder at `root` for the store `store` (a JMAP account id), with its state in `state` (made if missing).
101    pub fn open(root: impl Into<PathBuf>, store: &str, state: &Path) -> Result<Folder> {
102        let root = root.into();
103        std::fs::create_dir_all(&root)?;
104        let state = State::open(state)?;
105        match state.meta("store")? {
106            Some(known) if known != store => {
107                return Err(Error::State(format!("this state belongs to the store {known}, not {store}")));
108            }
109            Some(_) => {}
110            None => state.set_meta("store", store)?,
111        }
112        Ok(Folder { root, store: store.to_owned(), state })
113    }
114
115    pub fn root(&self) -> &Path {
116        &self.root
117    }
118
119    /// One round: reads what changed on both sides and brings them into step.
120    pub async fn sync(&mut self, client: &Client) -> Result<Report> {
121        let mut report = Report::default();
122        let mut nodes = self.read_store(client).await?;
123        let remote = tree(&nodes, &mut report.skipped);
124        let local = scan(&self.root)?;
125        let mut base = self.state.base()?;
126        let actions = plan::plan(&base, &paths(&remote), &local);
127        let mut trash = None;
128        // Paths changed on the server by this round: their node is read again at the end.
129        let mut touched: Vec<RelPath> = Vec::new();
130        for action in actions {
131            let path = action_path(&action).to_owned();
132            match self.run(client, &action, &remote, &mut base, &mut trash, &mut touched).await {
133                Ok(()) => count(&mut report, &action),
134                Err(error) => report.failed.push((path, error.to_string())),
135            }
136        }
137        // Read the store again: what this round made there gets the server's ids and blobs. Changes by others in
138        // the meantime stay with their old base, so the next round sees them.
139        if !touched.is_empty() {
140            nodes = self.read_store(client).await?;
141            let after = tree(&nodes, &mut Vec::new());
142            for path in touched {
143                if let (Some((node, _)), Some(entry)) = (after.get(&path), base.get_mut(&path)) {
144                    entry.remote = remote_of(node);
145                }
146            }
147        }
148        self.state.set_base(&base)?;
149        Ok(report)
150    }
151
152    /// The store's tree, brought up to date with its changes – or read whole.
153    async fn read_store(&mut self, client: &Client) -> Result<BTreeMap<String, Node>> {
154        let mut nodes = self.state.remote()?;
155        let mut since = self.state.meta("state")?.filter(|state| !state.is_empty());
156        loop {
157            let Some(state) = since.clone() else {
158                let snapshot = files::snapshot(client, &self.store).await?;
159                nodes = snapshot.nodes.iter().map(|node| (node.id.clone(), node_of(node))).collect();
160                self.state.set_remote(&nodes, &snapshot.state)?;
161                return Ok(nodes);
162            };
163            match files::changes(client, &self.store, &state, MAX_CHANGES).await {
164                Ok(changes) => {
165                    for id in &changes.destroyed {
166                        nodes.remove(id);
167                    }
168                    for node in &changes.changed {
169                        nodes.insert(node.id.clone(), node_of(node));
170                    }
171                    self.state.set_remote(&nodes, &changes.new_state)?;
172                    if !changes.has_more_changes {
173                        return Ok(nodes);
174                    }
175                    since = Some(changes.new_state);
176                }
177                Err(kernoid_client::Error::Method { kind, .. }) if kind == "cannotCalculateChanges" => since = None,
178                Err(error) => return Err(error.into()),
179            }
180        }
181    }
182
183    async fn run(
184        &self,
185        client: &Client,
186        action: &Action,
187        remote: &BTreeMap<RelPath, (Node, Remote)>,
188        base: &mut BTreeMap<RelPath, Base>,
189        trash: &mut Option<Option<String>>,
190        touched: &mut Vec<RelPath>,
191    ) -> Result<()> {
192        match action {
193            Action::KeepLocalCopy(path) => {
194                let from = self.local(path);
195                tokio::fs::rename(&from, conflict_copy(&from)).await?;
196                base.remove(path);
197            }
198            Action::DeleteLocal(path) => {
199                let at = self.local(path);
200                match tokio::fs::symlink_metadata(&at).await {
201                    Ok(meta) if meta.is_dir() => tokio::fs::remove_dir_all(&at).await?,
202                    Ok(_) => tokio::fs::remove_file(&at).await?,
203                    Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
204                    Err(error) => return Err(error.into()),
205                }
206                forget(base, path);
207            }
208            Action::MakeLocalDirectory(path) => {
209                let at = self.local(path);
210                tokio::fs::create_dir_all(&at).await?;
211                let (_, node) = remote.get(path).ok_or_else(|| gone(path))?;
212                base.insert(path.clone(), Base { remote: node.clone(), local: stat(&at)? });
213            }
214            Action::Download { path, id, blob } => {
215                let at = self.local(path);
216                let partial = PathBuf::from(format!("{}{PARTIAL}", at.display()));
217                let mut file = tokio::fs::File::create(&partial).await?;
218                let name = path.rsplit('/').next().unwrap_or(path);
219                let result = client
220                    .download(&self.store, blob, name, "application/octet-stream", &mut file, &mut |_, _| {})
221                    .await;
222                if let Err(error) = result {
223                    let _ = tokio::fs::remove_file(&partial).await;
224                    return Err(error.into());
225                }
226                file.sync_all().await?;
227                drop(file);
228                tokio::fs::rename(&partial, &at).await?;
229                let (_, node) = remote.get(path).filter(|(n, _)| &n.id == id).ok_or_else(|| gone(path))?;
230                base.insert(path.clone(), Base { remote: node.clone(), local: stat(&at)? });
231            }
232            Action::MakeRemoteDirectory(path) => {
233                let (parent, name) = self.parent_and_name(path, remote, base)?;
234                let create = json!({ "create": { "d": { "@type": "Directory", "parentId": parent, "name": name } } });
235                let result = client.set(&[FILENODE], "FileNode", &self.store, create, json!({})).await?;
236                let id = created_id(&result)?;
237                let local = stat(&self.local(path))?;
238                base.insert(path.clone(), Base { remote: Remote { id, directory: true, blob: None, size: 0 }, local });
239                touched.push(path.clone());
240            }
241            Action::Upload { path, id } => {
242                let at = self.local(path);
243                // The state of the file before it is read: a change while it travels shows in the next round.
244                let local = stat(&at)?;
245                let name = path.rsplit('/').next().unwrap_or(path).to_owned();
246                let kind = files::media_type(&name);
247                let file = tokio::fs::File::open(&at).await?;
248                let upload = client.upload(&self.store, kind, file, local.size, |_, _| {}).await?;
249                let node_id = match id {
250                    Some(id) => {
251                        let update =
252                            json!({ "update": { id.as_str(): { "blobId": upload.blob_id, "mediaType": kind } } });
253                        client.set(&[FILENODE], "FileNode", &self.store, update, json!({})).await?;
254                        id.clone()
255                    }
256                    None => {
257                        let (parent, _) = self.parent_and_name(path, remote, base)?;
258                        let create = json!({ "create": { "f": {
259                            "@type": "File", "parentId": parent, "name": name, "blobId": upload.blob_id,
260                            "mediaType": kind,
261                        } } });
262                        let result = client.set(&[FILENODE], "FileNode", &self.store, create, json!({})).await?;
263                        created_id(&result)?
264                    }
265                };
266                let remote = Remote { id: node_id, directory: false, blob: Some(upload.blob_id), size: upload.size };
267                base.insert(path.clone(), Base { remote, local });
268                touched.push(path.clone());
269            }
270            Action::DeleteRemote { path, id } => {
271                if trash.is_none() {
272                    *trash = Some(files::trash(client, &self.store).await?.map(|node| node.id));
273                }
274                match trash.as_ref().and_then(Option::as_ref) {
275                    Some(trash) => files::move_to(client, &self.store, id, Some(trash)).await?,
276                    None => files::destroy(client, &self.store, id).await?,
277                }
278                forget(base, path);
279            }
280            Action::Adopt(path) => {
281                let (_, node) = remote.get(path).ok_or_else(|| gone(path))?;
282                base.insert(path.clone(), Base { remote: node.clone(), local: stat(&self.local(path))? });
283            }
284        }
285        Ok(())
286    }
287
288    fn local(&self, path: &str) -> PathBuf {
289        path.split('/').fold(self.root.clone(), |at, part| at.join(part))
290    }
291
292    /// The id of a path's parent in the store – `None` at the top – and its name.
293    fn parent_and_name(
294        &self,
295        path: &str,
296        remote: &BTreeMap<RelPath, (Node, Remote)>,
297        base: &BTreeMap<RelPath, Base>,
298    ) -> Result<(Option<String>, String)> {
299        let Some((parent, name)) = path.rsplit_once('/') else { return Ok((None, path.to_owned())) };
300        let id = base
301            .get(parent)
302            .map(|entry| entry.remote.id.clone())
303            .or_else(|| remote.get(parent).map(|(node, _)| node.id.clone()))
304            .ok_or_else(|| gone(parent))?;
305        Ok((Some(id), name.to_owned()))
306    }
307}
308
309fn gone(path: &str) -> Error {
310    Error::State(format!("{path}: not in the store (any more)"))
311}
312
313fn action_path(action: &Action) -> &str {
314    match action {
315        Action::MakeLocalDirectory(path)
316        | Action::MakeRemoteDirectory(path)
317        | Action::DeleteLocal(path)
318        | Action::KeepLocalCopy(path)
319        | Action::Adopt(path) => path,
320        Action::Download { path, .. } | Action::Upload { path, .. } | Action::DeleteRemote { path, .. } => path,
321    }
322}
323
324fn count(report: &mut Report, action: &Action) {
325    match action {
326        Action::Download { .. } => report.downloaded += 1,
327        Action::Upload { .. } => report.uploaded += 1,
328        Action::DeleteLocal(_) => report.deleted_here += 1,
329        Action::DeleteRemote { .. } => report.deleted_there += 1,
330        Action::MakeLocalDirectory(_) | Action::MakeRemoteDirectory(_) => report.directories += 1,
331        Action::KeepLocalCopy(path) => report.conflicts.push(path.clone()),
332        Action::Adopt(_) => {}
333    }
334}
335
336/// Forgets a path and what lies below it.
337fn forget(base: &mut BTreeMap<RelPath, Base>, path: &str) {
338    let prefix = format!("{path}/");
339    base.retain(|known, _| known != path && !known.starts_with(&prefix));
340}
341
342fn created_id(result: &Value) -> Result<String> {
343    result
344        .get("created")
345        .and_then(Value::as_object)
346        .and_then(|created| created.values().next())
347        .and_then(|node| node.get("id"))
348        .and_then(Value::as_str)
349        .map(str::to_owned)
350        .ok_or_else(|| Error::Client(kernoid_client::Error::Protocol("no id for what was made".into())))
351}
352
353fn node_of(node: &FileNode) -> Node {
354    Node {
355        id: node.id.clone(),
356        parent: node.parent_id.clone(),
357        name: node.name.clone(),
358        directory: node.is_directory(),
359        blob: node.blob_id.clone(),
360        size: node.size.unwrap_or_default(),
361        role: node.role.clone(),
362    }
363}
364
365fn remote_of(node: &Node) -> Remote {
366    Remote { id: node.id.clone(), directory: node.directory, blob: node.blob.clone(), size: node.size }
367}
368
369/// Whether a name of the store can be a name in a folder.
370fn usable(name: &str) -> bool {
371    !name.is_empty() && name != "." && name != ".." && !name.contains(['/', '\\', '\0']) && !name.ends_with(PARTIAL)
372}
373
374/// The store's nodes by path – without the trash and what is in it, and without names a folder cannot hold (they
375/// go to `skipped`, with what is below them).
376pub fn tree(nodes: &BTreeMap<String, Node>, skipped: &mut Vec<String>) -> BTreeMap<RelPath, (Node, Remote)> {
377    let mut out = BTreeMap::new();
378    for node in nodes.values() {
379        let mut parts = Vec::new();
380        let mut at = Some(node);
381        let mut ok = true;
382        while let Some(current) = at {
383            if current.role.as_deref() == Some("trash") || !usable(&current.name) || parts.len() > 64 {
384                ok = false;
385                if !usable(&current.name) && current.id == node.id {
386                    skipped.push(node.name.clone());
387                }
388                break;
389            }
390            parts.push(current.name.as_str());
391            at = current.parent.as_ref().and_then(|parent| nodes.get(parent));
392            if at.is_none() && current.parent.is_some() {
393                // A parent the user cannot see: the node is not reachable from the top.
394                ok = false;
395            }
396        }
397        if ok {
398            parts.reverse();
399            out.insert(parts.join("/"), (node.clone(), remote_of(node)));
400        }
401    }
402    out
403}
404
405fn paths(tree: &BTreeMap<RelPath, (Node, Remote)>) -> BTreeMap<RelPath, Remote> {
406    tree.iter().map(|(path, (_, remote))| (path.clone(), remote.clone())).collect()
407}
408
409fn stat(path: &Path) -> Result<Local> {
410    let meta = std::fs::symlink_metadata(path)?;
411    let modified = meta
412        .modified()
413        .ok()
414        .and_then(|time| time.duration_since(UNIX_EPOCH).ok())
415        .map_or(0, |since| since.as_nanos() as i128);
416    Ok(Local { directory: meta.is_dir(), size: if meta.is_dir() { 0 } else { meta.len() }, modified })
417}
418
419/// The files and directories of a folder by path; links, unfinished downloads and other kinds are left out.
420pub fn scan(root: &Path) -> Result<BTreeMap<RelPath, Local>> {
421    let mut out = BTreeMap::new();
422    let mut stack = vec![(root.to_path_buf(), String::new())];
423    while let Some((dir, prefix)) = stack.pop() {
424        for entry in std::fs::read_dir(&dir)? {
425            let entry = entry?;
426            let Some(name) = entry.file_name().to_str().map(str::to_owned) else { continue };
427            if !usable(&name) {
428                continue;
429            }
430            let kind = entry.file_type()?;
431            if !kind.is_dir() && !kind.is_file() {
432                continue;
433            }
434            let path = if prefix.is_empty() { name } else { format!("{prefix}/{name}") };
435            out.insert(path.clone(), stat(&entry.path())?);
436            if kind.is_dir() {
437                stack.push((entry.path(), path));
438            }
439        }
440    }
441    Ok(out)
442}
443
444/// `report (conflict 2026-10-30 1412).txt` beside `report.txt` – a name not taken yet.
445pub fn conflict_copy(path: &Path) -> PathBuf {
446    let stem = path.file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default();
447    let extension = path.extension().map(|e| format!(".{}", e.to_string_lossy())).unwrap_or_default();
448    let when = chrono::Local::now().format("%Y-%m-%d %H%M");
449    let mut candidate = path.with_file_name(format!("{stem} (conflict {when}){extension}"));
450    let mut n = 2;
451    while candidate.exists() {
452        candidate = path.with_file_name(format!("{stem} (conflict {when} {n}){extension}"));
453        n += 1;
454    }
455    candidate
456}
457
458#[cfg(test)]
459mod tests {
460    use super::*;
461
462    fn node(id: &str, parent: Option<&str>, name: &str, directory: bool) -> (String, Node) {
463        (
464            id.into(),
465            Node {
466                id: id.into(),
467                parent: parent.map(Into::into),
468                name: name.into(),
469                directory,
470                blob: (!directory).then(|| format!("b-{id}")),
471                size: 1,
472                role: None,
473            },
474        )
475    }
476
477    #[test]
478    fn the_tree_by_path_without_the_trash_and_bad_names() {
479        let mut nodes: BTreeMap<String, Node> = [
480            node("1", None, "Projects", true),
481            node("2", Some("1"), "plan.txt", false),
482            node("3", None, "Trash", true),
483            node("4", Some("3"), "old.txt", false),
484            node("5", None, "..", false),
485            node("6", Some("9"), "orphan.txt", false),
486        ]
487        .into_iter()
488        .collect();
489        nodes.get_mut("3").unwrap().role = Some("trash".into());
490        let mut skipped = Vec::new();
491        let tree = tree(&nodes, &mut skipped);
492        assert_eq!(tree.keys().collect::<Vec<_>>(), ["Projects", "Projects/plan.txt"]);
493        assert_eq!(skipped, [".."]);
494    }
495
496    #[test]
497    fn a_folder_scanned_and_a_conflict_copy_named() {
498        let root = std::env::temp_dir().join(format!("kernoid-sync-scan-{}", std::process::id()));
499        std::fs::create_dir_all(root.join("D")).unwrap();
500        std::fs::write(root.join("D/a.txt"), "abc").unwrap();
501        std::fs::write(root.join(format!("b.txt{PARTIAL}")), "half").unwrap();
502        let found = scan(&root).unwrap();
503        assert_eq!(found.keys().collect::<Vec<_>>(), ["D", "D/a.txt"]);
504        assert_eq!((found["D/a.txt"].size, found["D"].directory), (3, true));
505        let copy = conflict_copy(&root.join("D/a.txt"));
506        let name = copy.file_name().unwrap().to_string_lossy().into_owned();
507        assert!(name.starts_with("a (conflict ") && name.ends_with(").txt"), "{name}");
508        std::fs::remove_dir_all(&root).unwrap();
509    }
510}
kernoid_sync/lib.rs

kernoid_sync/
lib.rs
