semora_cli/commands/

pull.rs

//! `semora pull` and `semora status`: a project's documents on the server, against this folder.
//!
//! Both compare three things for each document path: what the server has now, what this
//! checkout last pulled or pushed (its record), and what is on disk. `status` prints that;
//! `pull` writes what is newer on the server and never overwrites a file changed here unless
//! told to.

use std::collections::{BTreeMap, BTreeSet};
use std::io::ErrorKind;
use std::path::{Path, PathBuf};

use anyhow::{Context, bail};
use semora_proto::{DocDetail, DocKind, DocSummary};
use serde::Serialize;

use crate::args::PullArgs;
use crate::checkout::{self, Local, Record};
use crate::client::Client;
use crate::project_link::{self, Link, shown};
use crate::{Ctx, exit, output};

/// Where a document stands, here against the server.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum State {
    /// The same here and on the server.
    Clean,
    /// Changed here since the last pull or push; nothing newer on the server.
    Modified,
    /// Newer on the server; unchanged here.
    Behind,
    /// On the server, not here.
    Missing,
    /// Changed here and on the server.
    Diverged,
    /// Here and on the server and different, and this folder never pulled or pushed it.
    Untracked,
    /// Deleted on the server; the copy here is unchanged.
    Deleted,
    /// Deleted on the server, and changed here.
    DeletedModified,
    /// It can't be a file or folder here.
    Blocked,
    /// Deleted on the server and gone here: only the record is left.
    #[serde(skip)]
    Gone,
}

impl State {
    fn label(self) -> &'static str {
        match self {
            State::Clean => "same",
            State::Modified => "changed here",
            State::Behind => "newer on the server",
            State::Missing => "not here",
            State::Diverged => "changed on both",
            State::Untracked => "different, never pulled",
            State::Deleted | State::DeletedModified | State::Gone => "deleted on the server",
            State::Blocked => "can't be here",
        }
    }
}

/// One document path, as `status` reports it.
#[derive(Debug, Serialize)]
pub struct Entry {
    pub path: String,
    pub state: State,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub id: Option<String>,
    /// The revision this folder last pulled or pushed.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub here: Option<i64>,
    /// The revision on the server now.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub server: Option<i64>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub reason: Option<String>,
}

/// A document path with everything known about it.
struct Item {
    key: String,
    on_server: Option<DocSummary>,
    record: Option<Record>,
    state: State,
    reason: Option<String>,
    /// The server's detail, when classifying needed it.
    detail: Option<DocDetail>,
}

impl Item {
    fn entry(&self) -> Entry {
        Entry {
            path: self.key.clone(),
            state: self.state,
            id: self
                .on_server
                .as_ref()
                .map(|s| s.id.clone())
                .or_else(|| self.record.as_ref().map(|r| r.id.clone())),
            here: self.record.as_ref().map(|r| r.rev),
            server: self
                .detail
                .as_ref()
                .map(|d| d.summary.rev)
                .or_else(|| self.on_server.as_ref().map(|s| s.rev)),
            reason: self.reason.clone(),
        }
    }
}

/// The project folder for `path` (default: the current folder), and the document path it
/// narrows to (`None`: the whole project). The path need not exist yet.
fn scope(path: Option<&Path>) -> anyhow::Result<(project_link::FileLink, Option<String>)> {
    let start = match path {
        Some(p) => std::path::absolute(p).with_context(|| format!("cannot read {}", shown(p)))?,
        None => std::env::current_dir().context("reading the current folder")?,
    };
    // The nearest folder that exists: a path that was never pulled has none of its own.
    let mut existing = start.as_path();
    let mut tail = Vec::new();
    while !existing.exists() {
        let name = existing
            .file_name()
            .with_context(|| format!("{} does not exist", shown(&start)))?;
        tail.push(name.to_owned());
        existing = existing
            .parent()
            .with_context(|| format!("{} does not exist", shown(&start)))?;
    }
    let mut full = std::fs::canonicalize(existing)
        .with_context(|| format!("cannot read {}", shown(existing)))?;
    let link = project_link::find(&full)?.with_context(|| {
        format!(
            "{} is not in a project: no {} here or in a parent folder. Run `semora project init` to make one.",
            shown(&full),
            project_link::FILE
        )
    })?;
    for name in tail.into_iter().rev() {
        full.push(name);
    }
    let rel = full.strip_prefix(&link.root).with_context(|| {
        format!(
            "{} is not inside the project at {}",
            shown(&full),
            shown(&link.root)
        )
    })?;
    let mut parts = Vec::new();
    for c in rel.components() {
        let s = c
            .as_os_str()
            .to_str()
            .with_context(|| format!("{} isn't valid Unicode", shown(&full)))?;
        parts.push(s.to_string());
    }
    Ok((link, (!parts.is_empty()).then(|| parts.join("/"))))
}

/// A document is in scope when it is at the path, under it, or holds it (a bundle).
fn in_scope(key: &str, prefix: Option<&str>) -> bool {
    match prefix {
        None => true,
        Some(p) => {
            key == p
                || key.strip_prefix(p).is_some_and(|r| r.starts_with('/'))
                || p.strip_prefix(key).is_some_and(|r| r.starts_with('/'))
        }
    }
}

/// The server's files of a document, by path from the project folder, with their hashes.
fn server_files(key: &str, d: &DocDetail) -> Result<BTreeMap<String, String>, String> {
    let mut out = BTreeMap::new();
    if d.summary.kind == DocKind::Bundle {
        for f in &d.files {
            let local = format!("{key}/{}", f.path);
            if checkout::local_rel(&local).is_none() {
                return Err(format!(
                    "the server lists a file `{}` that can't be written here",
                    f.path.escape_default()
                ));
            }
            out.insert(local, f.hash.clone());
        }
    } else {
        let entry = d
            .files
            .iter()
            .find(|f| f.path == d.entry_path)
            .or(d.files.first())
            .ok_or_else(|| "the server lists no file for it".to_string())?;
        out.insert(key.to_string(), entry.hash.clone());
    }
    Ok(out)
}

/// What a folder or file of the wrong type says.
fn wrong_type(kind: DocKind) -> String {
    if kind == DocKind::Bundle {
        "it is a folder on the server, and a file is here".into()
    } else {
        "it is a file on the server, and a folder is here (or its path runs through a file)".into()
    }
}

/// Every document path in scope, classified.
async fn survey(client: &Client, link: &Link, prefix: Option<&str>) -> anyhow::Result<Vec<Item>> {
    let docs = client.project_docs(&link.slug).await?;
    let records = checkout::load(link)?;
    let on_server: BTreeMap<String, DocSummary> = docs
        .into_iter()
        .filter_map(|d| Some((d.path.clone()?, d)))
        .collect();
    let keys: BTreeSet<&String> = on_server
        .keys()
        .chain(records.keys())
        .filter(|k| in_scope(k, prefix))
        .collect();
    let mut items = Vec::new();
    for key in keys {
        let s = on_server.get(key).cloned();
        let r = records.get(key).cloned();
        let mut item = Item {
            key: key.clone(),
            on_server: s.clone(),
            record: r.clone(),
            state: State::Clean,
            reason: None,
            detail: None,
        };
        let kind = s.as_ref().map(|s| s.kind).or(r.as_ref().map(|r| r.kind));
        let local = checkout::local(&link.root, key, kind.expect("on the server or recorded"))?;
        let Some(s) = s else {
            let r = r.expect("recorded");
            item.state = match local {
                Local::Missing => State::Gone,
                Local::Files(f) if f == r.files => State::Deleted,
                _ => State::DeletedModified,
            };
            items.push(item);
            continue;
        };
        // A document whose path holds others can't be a file or a folder of its own here.
        let prefix_of = format!("{key}/");
        if let Some(inner) = on_server.keys().find(|k| k.starts_with(&prefix_of)) {
            item.state = State::Blocked;
            item.reason = Some(format!(
                "`{inner}` is a document of its own inside its path, so it can't be a {} here",
                if s.kind == DocKind::Bundle {
                    "folder"
                } else {
                    "file"
                }
            ));
            items.push(item);
            continue;
        }
        if checkout::local_rel(key).is_none() {
            item.state = State::Blocked;
            item.reason = Some("its path can't be a file name here".into());
            items.push(item);
            continue;
        }
        item.state = match (local, &r) {
            (Local::WrongType, _) => {
                item.reason = Some(wrong_type(s.kind));
                State::Blocked
            }
            (Local::Missing, _) => State::Missing,
            (Local::Files(f), Some(r)) => {
                let same = r.id == s.id && r.rev == s.rev;
                match (same, f == r.files) {
                    (true, true) => State::Clean,
                    (true, false) => State::Modified,
                    (false, true) => State::Behind,
                    (false, false) => State::Diverged,
                }
            }
            (Local::Files(f), None) => {
                // Never pulled or pushed here: compare with the server's files.
                let d = client.doc(&s.id).await?;
                let state = match server_files(key, &d) {
                    Ok(theirs) if theirs == f => State::Clean,
                    Ok(_) => State::Untracked,
                    Err(why) => {
                        item.reason = Some(why);
                        State::Blocked
                    }
                };
                item.detail = Some(d);
                state
            }
        };
        items.push(item);
    }
    Ok(items)
}

/// The client, the project (under the id the server has for it now) and the path prefix.
async fn project_and_scope(
    ctx: &Ctx,
    path: Option<&Path>,
) -> anyhow::Result<(Client, Link, Option<String>)> {
    let client = ctx.client()?;
    let (file, prefix) = scope(path)?;
    let link = project_link::resolve(&client, &ctx.settings.config_dir, file).await?;
    let link = project_link::follow(&client, link).await?;
    Ok((client, link, prefix))
}

pub async fn status(ctx: &Ctx, path: Option<PathBuf>) -> anyhow::Result<i32> {
    let (client, link, prefix) = project_and_scope(ctx, path.as_deref()).await?;
    let items = survey(&client, &link, prefix.as_deref()).await?;
    let entries: Vec<Entry> = items
        .iter()
        .filter(|i| i.state != State::Gone)
        .map(Item::entry)
        .collect();
    if ctx.json {
        output::json(&serde_json::json!({
            "project": link.slug,
            "folder": shown(&link.root),
            "documents": entries,
        }));
        return Ok(exit::OK);
    }
    let open: Vec<&Entry> = entries.iter().filter(|e| e.state != State::Clean).collect();
    println!("Project {} in {}", link.slug, shown(&link.root));
    if open.is_empty() {
        println!(
            "Everything here matches the server ({}).",
            count(entries.len(), "document")
        );
        return Ok(exit::OK);
    }
    let width = open
        .iter()
        .map(|e| e.state.label().len())
        .max()
        .unwrap_or(0);
    for e in &open {
        let revs = match (e.state, e.here, e.server) {
            (State::Modified, Some(h), _) => format!("  (from r{h})"),
            (State::Behind, Some(h), Some(s)) => format!("  (r{h} here, r{s} on the server)"),
            (State::Diverged, Some(h), Some(s)) => format!("  (from r{h}; the server has r{s})"),
            (State::Missing | State::Untracked, _, Some(s)) => format!("  (r{s})"),
            (State::DeletedModified, _, _) => "  (changed here)".into(),
            _ => String::new(),
        };
        let why = e
            .reason
            .as_deref()
            .map(|r| format!(": {r}"))
            .unwrap_or_default();
        println!("  {:<width$}  {}{revs}{why}", e.state.label(), e.path);
    }
    let pull = open.iter().any(|e| {
        matches!(
            e.state,
            State::Behind | State::Missing | State::Deleted | State::Diverged | State::Untracked
        )
    });
    let push = open
        .iter()
        .any(|e| matches!(e.state, State::Modified | State::Diverged));
    match (pull, push) {
        (true, true) => println!(
            "Pull to get what is newer on the server; merge what changed on both before you push it."
        ),
        (true, false) => println!("Run `semora pull` to get what is newer on the server."),
        (false, true) => println!("Push what changed here."),
        (false, false) => {}
    }
    Ok(exit::OK)
}

fn count(n: usize, what: &str) -> String {
    if n == 1 {
        format!("1 {what}")
    } else {
        format!("{n} {what}s")
    }
}

/// What `pull` did with one document.
#[derive(Debug, Serialize)]
struct Done {
    path: String,
    #[serde(skip_serializing_if = "Option::is_none")]
    id: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    rev: Option<i64>,
    /// Why it was skipped, or that it changed here and isn't pushed yet.
    #[serde(skip_serializing_if = "Option::is_none")]
    note: Option<String>,
}

#[derive(Debug, Default, Serialize)]
struct Pulled {
    written: Vec<Done>,
    unchanged: Vec<Done>,
    skipped: Vec<Done>,
    deleted: Vec<Done>,
}

/// Writes `data` to `path` through a hidden file beside it, so a reader never sees half of it.
fn write_file(path: &Path, data: &[u8]) -> anyhow::Result<()> {
    let dir = path.parent().expect("a path in the project folder");
    std::fs::create_dir_all(dir).with_context(|| format!("creating {}", shown(dir)))?;
    let name = path
        .file_name()
        .expect("a file name")
        .to_string_lossy()
        .into_owned();
    let tmp = dir.join(format!(".{name}.{:08x}.semora-tmp", rand::random::<u32>()));
    std::fs::write(&tmp, data).with_context(|| format!("writing {}", shown(&tmp)))?;
    if let Err(e) = std::fs::rename(&tmp, path) {
        let _ = std::fs::remove_file(&tmp);
        return Err(e).with_context(|| format!("writing {}", shown(path)));
    }
    Ok(())
}

/// The sha256 of the file at `path`, if there is one.
fn hash_of(path: &Path) -> Option<String> {
    std::fs::read(path).ok().map(|d| checkout::sha256(&d))
}

/// Removes `rel` from the project folder, then the folders above it inside `stop` that it left
/// empty.
fn remove_tracked(root: &Path, rel: &str, stop: &Path) -> anyhow::Result<()> {
    let Some(p) = checkout::local_rel(rel) else {
        return Ok(());
    };
    let full = root.join(p);
    match std::fs::remove_file(&full) {
        Ok(()) => {}
        Err(e) if e.kind() == ErrorKind::NotFound => {}
        Err(e) => return Err(e).with_context(|| format!("removing {}", shown(&full))),
    }
    let mut dir = full.parent();
    while let Some(d) = dir {
        if !d.starts_with(stop) || std::fs::remove_dir(d).is_err() {
            break;
        }
        dir = d.parent();
    }
    Ok(())
}

/// Writes the server's current revision of a document into the folder. Files a newer revision
/// dropped go too, when they are as this folder last had them. Returns the new record, or why it
/// was skipped.
async fn write_doc(
    client: &Client,
    root: &Path,
    item: &Item,
) -> anyhow::Result<Result<Record, String>> {
    let s = item.on_server.as_ref().expect("on the server");
    let d = match &item.detail {
        Some(d) => d.clone(),
        None => client.doc(&s.id).await?,
    };
    let theirs = match server_files(&item.key, &d) {
        Ok(t) => t,
        Err(why) => return Ok(Err(why)),
    };
    let rev = d.summary.rev;
    for (local, hash) in &theirs {
        let path = root.join(checkout::local_rel(local).expect("checked by server_files"));
        if hash_of(&path).as_deref() == Some(hash.as_str()) {
            continue;
        }
        let in_doc = local
            .strip_prefix(&item.key)
            .and_then(|r| r.strip_prefix('/'))
            .filter(|_| d.summary.kind == DocKind::Bundle);
        let data = client.raw(&d.summary.id, in_doc, Some(rev)).await?;
        if checkout::sha256(&data) != *hash {
            bail!(
                "the server sent {local} with other content than it listed; nothing of it was written. Try again."
            );
        }
        write_file(&path, &data)?;
    }
    if let Some(r) = &item.record {
        let stop = root.join(checkout::local_rel(&item.key).expect("checked"));
        for (old, hash) in &r.files {
            if theirs.contains_key(old) {
                continue;
            }
            let Some(rel) = checkout::local_rel(old) else {
                continue;
            };
            if hash_of(&root.join(rel)).as_deref() == Some(hash.as_str()) {
                remove_tracked(root, old, &stop)?;
            }
        }
    }
    Ok(Ok(Record {
        id: d.summary.id.clone(),
        rev,
        kind: d.summary.kind,
        files: theirs,
    }))
}

pub async fn pull(ctx: &Ctx, args: PullArgs) -> anyhow::Result<i32> {
    let (client, link, prefix) = project_and_scope(ctx, args.path.as_deref()).await?;
    let items = survey(&client, &link, prefix.as_deref()).await?;
    let root = link.root.clone();
    let mut out = Pulled::default();
    for item in &items {
        let e = item.entry();
        let done = |note: Option<String>| Done {
            path: e.path.clone(),
            id: e.id.clone(),
            rev: e.server.or(e.here),
            note,
        };
        let write = match item.state {
            State::Behind | State::Missing => true,
            State::Diverged | State::Untracked => args.force,
            _ => false,
        };
        if write {
            match write_doc(&client, &root, item).await? {
                Ok(record) => {
                    let rev = record.rev;
                    checkout::update(&link, |r| r.insert(item.key.clone(), record))?;
                    out.written.push(Done {
                        rev: Some(rev),
                        ..done(None)
                    });
                }
                Err(why) => out.skipped.push(done(Some(why))),
            }
            continue;
        }
        match item.state {
            State::Clean => {
                // Already the same; a folder that never pulled it records it now.
                if item.record.is_none()
                    && let Some(d) = &item.detail
                    && let Ok(files) = server_files(&item.key, d)
                {
                    let record = Record {
                        id: d.summary.id.clone(),
                        rev: d.summary.rev,
                        kind: d.summary.kind,
                        files,
                    };
                    checkout::update(&link, |r| r.insert(item.key.clone(), record))?;
                }
                out.unchanged.push(done(None));
            }
            State::Modified => out
                .unchanged
                .push(done(Some("changed here, not pushed yet".into()))),
            State::Diverged => out.skipped.push(done(Some(format!(
                "changed here and on the server (this folder has r{}, the server r{}). Merge the server's copy into it and push with --force, or pull --force to take the server's",
                e.here.unwrap_or_default(),
                e.server.unwrap_or_default()
            )))),
            State::Untracked => out.skipped.push(done(Some(
                "already here and different from the server's, and this folder never pulled or pushed it. pull --force takes the server's; push --force keeps yours".into(),
            ))),
            State::Deleted => {
                let r = item.record.as_ref().expect("recorded");
                let stop = root.join(checkout::local_rel(&item.key).unwrap_or_default());
                for f in r.files.keys() {
                    remove_tracked(&root, f, &stop)?;
                }
                checkout::update(&link, |recs| recs.remove(&item.key))?;
                out.deleted.push(done(None));
            }
            State::DeletedModified => out.skipped.push(done(Some(
                "deleted on the server, and changed here. Push it with --force to put it back, or delete it".into(),
            ))),
            State::Gone => {
                checkout::update(&link, |recs| recs.remove(&item.key))?;
            }
            State::Blocked => out.skipped.push(done(item.reason.clone())),
            State::Behind | State::Missing => unreachable!("written above"),
        }
    }
    if ctx.json {
        output::json(&out);
        return Ok(exit::OK);
    }
    print_pulled(&link, &out);
    Ok(exit::OK)
}

fn print_pulled(link: &Link, out: &Pulled) {
    let r = |d: &Done| d.rev.map(|r| format!(" (r{r})")).unwrap_or_default();
    if out.written.is_empty() && out.deleted.is_empty() && out.skipped.is_empty() {
        println!(
            "Nothing new in {}: {} the same as on the server.",
            link.slug,
            count(out.unchanged.len(), "document")
        );
    } else {
        let done = output::out().done;
        anstream::println!(
            "{done}Pulled{done:#} {} into {}",
            link.slug,
            shown(&link.root)
        );
        for d in &out.written {
            println!("  wrote    {}{}", d.path, r(d));
        }
        for d in &out.deleted {
            println!("  deleted  {}", d.path);
        }
        for d in &out.skipped {
            println!(
                "  skipped  {}: {}",
                d.path,
                d.note.as_deref().unwrap_or("skipped")
            );
        }
        if !out.unchanged.is_empty() {
            println!("  {} unchanged", count(out.unchanged.len(), "document"));
        }
    }
    let local: Vec<&str> = out
        .unchanged
        .iter()
        .filter(|d| d.note.is_some())
        .map(|d| d.path.as_str())
        .collect();
    if !local.is_empty() {
        println!("Changed here, not pushed yet: {}", local.join(", "));
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn scope_covers_the_path_what_is_under_it_and_a_bundle_that_holds_it() {
        assert!(in_scope("docs/plan.md", None));
        assert!(in_scope("docs/plan.md", Some("docs")));
        assert!(in_scope("docs/plan.md", Some("docs/plan.md")));
        assert!(in_scope("docs/site", Some("docs/site/index.md")));
        assert!(!in_scope("docs-old/plan.md", Some("docs")));
        assert!(!in_scope("docs/plan.md", Some("docs/plan")));
        assert!(!in_scope("notes.md", Some("docs")));
    }

    #[test]
    fn a_path_that_does_not_exist_yet_still_finds_its_project() {
        let d = tempfile::tempdir().unwrap();
        std::fs::write(
            d.path().join(".semora"),
            "server = \"https://a.example.com\"\nproject = \"p\"\n",
        )
        .unwrap();
        let (link, prefix) = scope(Some(&d.path().join("docs/new/plan.md"))).unwrap();
        assert_eq!(link.slug, "p");
        assert_eq!(prefix.as_deref(), Some("docs/new/plan.md"));
        let (_, whole) = scope(Some(d.path())).unwrap();
        assert_eq!(whole, None);
        let lonely = tempfile::tempdir().unwrap();
        let err = scope(Some(lonely.path())).unwrap_err().to_string();
        assert!(err.contains("project init"), "{err}");
    }
}