defter 0.5.1

//! `defter migrate`: from one library database (`defter.sqlite3`) to the control
//! database and a database per library.
//!
//! The library database moves to `libraries/u<id>.sqlite3` unchanged, its files
//! stay under `files/`, and the server's ID, the cached Zotero schema and the
//! profile settings move to the control database. The steps can be repeated:
//! a migration interrupted between them finishes when run again.

use std::path::{Path, PathBuf};

use rusqlite::{Connection, OpenFlags};

use crate::control::{self, LibraryRef};
use crate::db;
use crate::profile;
use crate::registry;

/// The data directory's layout.
#[derive(Debug, Clone)]
pub struct Layout {
    /// The library database from before: `defter.sqlite3`, or `--db`.
    pub legacy: PathBuf,
    pub control: PathBuf,
    pub libraries: PathBuf,
}

impl Layout {
    pub fn new(data_dir: &Path, legacy: PathBuf) -> Self {
        Layout {
            legacy,
            control: data_dir.join("control.sqlite3"),
            libraries: data_dir.join("libraries"),
        }
    }

    pub fn library(&self, library: LibraryRef) -> PathBuf {
        registry::path(&self.libraries, library)
    }
}

/// Where a data directory stands.
#[derive(Debug, PartialEq, Eq)]
pub enum State {
    /// Nothing yet: a new server, which sets itself up on start.
    New,
    /// One library database, from before: `defter migrate` moves it.
    Legacy,
    /// The control database is there but the library not yet moved.
    Interrupted,
    Migrated,
    /// Both a migrated layout and a library database from before, such as
    /// when an older defter ran after the migration: which is current must be
    /// decided by hand.
    Both,
    /// Libraries but no control database, which knows where their files are
    /// and the server's ID: it must be restored, not started afresh.
    ControlMissing,
}

/// Where the data directory stands, for the owner `user_id`.
pub fn state(layout: &Layout, user_id: u64) -> State {
    let library = layout.library(LibraryRef::User(user_id));
    match (
        layout.control.exists(),
        layout.legacy.exists(),
        library.exists(),
    ) {
        (false, true, false) => State::Legacy,
        (false, true, true) => State::Both,
        (false, false, _) if any_library(layout) => State::ControlMissing,
        (false, false, _) => State::New,
        (true, true, false) => State::Interrupted,
        (true, true, true) => State::Both,
        (true, false, _) => State::Migrated,
    }
}

/// Whether `libraries/` holds any library's database.
fn any_library(layout: &Layout) -> bool {
    std::fs::read_dir(&layout.libraries).is_ok_and(|entries| {
        entries.flatten().any(|entry| {
            let name = entry.file_name().to_string_lossy().to_string();
            name.strip_suffix(".sqlite3")
                .is_some_and(|stem| LibraryRef::parse(stem).is_some())
        })
    })
}

/// What the owner's account is made of.
#[derive(Debug, Clone)]
pub struct Owner {
    pub user_id: u64,
    pub username: String,
    /// `--orcid`, which outranks the one recorded at sign-in.
    pub orcid: Option<String>,
    /// `--oidc-allowed-orcids`: the others become invitations.
    pub allowed_orcids: Vec<String>,
}

/// What a migration does.
#[derive(Debug)]
pub struct Plan {
    pub state: State,
    pub version: i64,
    pub orcid: Option<String>,
    pub profile_hidden: bool,
    pub invites: Vec<String>,
    /// The `meta` rows that move to the control database.
    pub meta: Vec<(String, String)>,
}

/// The `meta` rows of a library from before that move to the control database.
const MOVED_META: &[&str] = &["server_id", "schema", "schema_etag", "schema_fetched"];

/// Work out what migrating `layout` does, changing nothing.
pub fn plan(layout: &Layout, owner: &Owner) -> Result<Plan, String> {
    let state = state(layout, owner.user_id);
    match state {
        State::New => {
            return Err(
                "nothing to migrate: a new server sets itself up when it starts".to_string(),
            );
        }
        State::Migrated => {
            return Err(format!(
                "already migrated: {} is in place",
                layout.control.display()
            ));
        }
        State::Both => return Err(both(layout, owner.user_id)),
        State::ControlMissing => return Err(control_missing(layout)),
        State::Legacy | State::Interrupted => {}
    }
    same_file_system(&layout.legacy, &layout.libraries)?;
    let conn = Connection::open_with_flags(&layout.legacy, OpenFlags::SQLITE_OPEN_READ_ONLY)
        .map_err(|e| format!("opening {}: {e}", layout.legacy.display()))?;
    let check: String = conn
        .query_row("PRAGMA quick_check", [], |r| r.get(0))
        .map_err(|e| format!("checking {}: {e}", layout.legacy.display()))?;
    if check != "ok" {
        return Err(format!("{} is damaged: {check}", layout.legacy.display()));
    }
    let version: i64 = conn
        .query_row("SELECT version FROM library WHERE id = 1", [], |r| r.get(0))
        .map_err(|e| format!("reading {}: {e}", layout.legacy.display()))?;
    let meta = |key: &str| db::get_meta(&conn, key).map_err(|e| e.to_string());
    let orcid = match &owner.orcid {
        Some(orcid) => Some(orcid.clone()),
        None => meta(profile::OWNER_KEY)?,
    };
    let profile_hidden = meta(profile::HIDDEN_KEY)?.as_deref() == Some("1");
    let mut moved = Vec::new();
    for key in MOVED_META {
        if let Some(value) = meta(key)? {
            moved.push((key.to_string(), value));
        }
    }
    let mut invites: Vec<String> = owner
        .allowed_orcids
        .iter()
        .filter(|o| Some(*o) != orcid.as_ref())
        .cloned()
        .collect();
    invites.sort();
    invites.dedup();
    Ok(Plan {
        state,
        version,
        orcid,
        profile_hidden,
        invites,
        meta: moved,
    })
}

/// Renaming cannot move a file to another file system: refuse before changing
/// anything, rather than stop halfway.
fn same_file_system(legacy: &Path, libraries: &Path) -> Result<(), String> {
    #[cfg(unix)]
    {
        use std::os::unix::fs::MetadataExt;
        let device = |path: &Path| std::fs::metadata(path).map(|m| m.dev()).ok();
        let target = libraries.parent().unwrap_or(libraries);
        if let (Some(from), Some(to)) = (device(legacy), device(target))
            && from != to
        {
            return Err(format!(
                "{} is on another file system than {}: move it there first (and point \
                 DEFTER_DB at it, or leave DEFTER_DB unset if it is now defter.sqlite3)",
                legacy.display(),
                target.display()
            ));
        }
    }
    let _ = (legacy, libraries);
    Ok(())
}

fn control_missing(layout: &Layout) -> String {
    format!(
        "{} is missing, but {} holds libraries: restore it with `defter restore --control` \
         rather than start afresh, which would lose the server's ID and where attachment \
         files are",
        layout.control.display(),
        layout.libraries.display()
    )
}

fn both(layout: &Layout, user_id: u64) -> String {
    format!(
        "both {} and {} exist. If an older defter ran after the migration, it wrote to \
         the first: move whichever is older aside, then start again",
        layout.legacy.display(),
        layout.library(LibraryRef::User(user_id)).display()
    )
}

/// Carry out `plan`. The server must not be running (see `backup::lock`).
pub fn apply(layout: &Layout, owner: &Owner, plan: &Plan) -> Result<(), String> {
    if plan.state == State::Legacy {
        create_control(layout, owner, plan)?;
    } else {
        // Interrupted: the control database must be this owner's.
        let conn = control::open_connection(&layout.control).map_err(|e| e.to_string())?;
        if control::user(&conn, owner.user_id)
            .map_err(|e| e.to_string())?
            .is_none()
        {
            return Err(format!(
                "{} has no user {} (DEFTER_USER_ID)",
                layout.control.display(),
                owner.user_id
            ));
        }
    }
    move_library(layout, owner.user_id)
}

fn create_control(layout: &Layout, owner: &Owner, plan: &Plan) -> Result<(), String> {
    // Written beside it and moved into place whole, so a control database is
    // never half made.
    let making = layout.control.with_extension("sqlite3.migrating");
    remove_database(&making)?;
    {
        let mut conn = control::open_connection(&making).map_err(|e| e.to_string())?;
        let tx = conn.transaction().map_err(|e| e.to_string())?;
        let mut user = registry::owner(owner.user_id, &owner.username, true);
        user.orcid = plan.orcid.clone();
        user.profile_hidden = plan.profile_hidden;
        control::insert_user(&tx, &user).map_err(|e| e.to_string())?;
        for (key, value) in &plan.meta {
            db::set_meta(&tx, key, value).map_err(|e| e.to_string())?;
        }
        control::note_version(&tx, LibraryRef::User(owner.user_id), plan.version)
            .map_err(|e| e.to_string())?;
        for orcid in &plan.invites {
            tx.execute(
                "INSERT INTO invite (orcid, invited_by, note, created_at)
                 VALUES (?1, ?2, 'allowed to sign in before the migration', ?3)",
                rusqlite::params![orcid, owner.user_id as i64, control::now()],
            )
            .map_err(|e| e.to_string())?;
        }
        tx.commit().map_err(|e| e.to_string())?;
        conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE); PRAGMA journal_mode = DELETE;")
            .map_err(|e| e.to_string())?;
    }
    std::fs::rename(&making, &layout.control)
        .map_err(|e| format!("moving {} into place: {e}", layout.control.display()))
}

/// Move the library database from before to `libraries/u<id>.sqlite3`, with
/// everything in its write-ahead log written into it first.
fn move_library(layout: &Layout, user_id: u64) -> Result<(), String> {
    {
        let conn = Connection::open(&layout.legacy)
            .map_err(|e| format!("opening {}: {e}", layout.legacy.display()))?;
        conn.busy_timeout(std::time::Duration::from_secs(30))
            .map_err(|e| e.to_string())?;
        conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);")
            .map_err(|e| format!("checkpointing {}: {e}", layout.legacy.display()))?;
    }
    std::fs::create_dir_all(&layout.libraries)
        .map_err(|e| format!("creating {}: {e}", layout.libraries.display()))?;
    let to = layout.library(LibraryRef::User(user_id));
    // Renaming would replace a database there, or put this one beside a stale log.
    for suffix in ["", "-wal", "-shm"] {
        let target = PathBuf::from(format!("{}{suffix}", to.display()));
        if target.exists() {
            return Err(format!(
                "{} is in the way: move it aside, then run `defter migrate` again",
                target.display()
            ));
        }
    }
    // The log goes along, in case closing left one, and before the database, so
    // the database never stands without its log.
    for suffix in ["-wal", "-shm", ""] {
        let from = PathBuf::from(format!("{}{suffix}", layout.legacy.display()));
        if from.exists() {
            let target = PathBuf::from(format!("{}{suffix}", to.display()));
            std::fs::rename(&from, &target)
                .map_err(|e| format!("moving {}: {e}", from.display()))?;
        }
    }
    Ok(())
}

fn remove_database(path: &Path) -> Result<(), String> {
    for suffix in ["", "-wal", "-shm", "-journal"] {
        let file = PathBuf::from(format!("{}{suffix}", path.display()));
        if file.exists() {
            std::fs::remove_file(&file).map_err(|e| format!("removing {}: {e}", file.display()))?;
        }
    }
    Ok(())
}

/// Why the server cannot start in this data directory, if it cannot.
pub fn refuse_start(layout: &Layout, user_id: u64) -> Option<String> {
    match state(layout, user_id) {
        State::New | State::Migrated => None,
        State::Legacy => Some(format!(
            "{} is from an earlier defter: stop here, back up, and run `defter migrate` \
             (see \"Upgrading from 0.5.0\" in the README)",
            layout.legacy.display()
        )),
        State::Interrupted => {
            Some("a migration was interrupted: run `defter migrate` again to finish it".to_string())
        }
        State::Both => Some(both(layout, user_id)),
        State::ControlMissing => Some(control_missing(layout)),
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::db::Db;
    use serde_json::json;

    fn owner() -> Owner {
        Owner {
            user_id: 1,
            username: "owner".into(),
            orcid: None,
            allowed_orcids: vec!["0000-0003-1419-2405".into(), "0000-0002-1825-0097".into()],
        }
    }

    /// A library from before, as a server of 0.5.0 left it, with an open writer.
    async fn legacy(dir: &Path) -> (Layout, std::sync::Arc<Db>) {
        let layout = Layout::new(dir, dir.join("defter.sqlite3"));
        let db = Db::open(&layout.legacy).unwrap();
        db.server_id().await.unwrap();
        db.write(|c| {
            db::set_meta(c, profile::OWNER_KEY, "0000-0003-1419-2405")?;
            db::set_meta(c, profile::HIDDEN_KEY, "1")?;
            db::set_meta(c, "schema", "{}")?;
            let items = vec![json!({"key": "AAAAAAAA", "itemType": "book", "title": "x"})];
            crate::store::write_batch(c, crate::store::Kind::Item, items, Some(0))?;
            Ok(())
        })
        .await
        .unwrap();
        (layout, db)
    }

    #[tokio::test]
    async fn moves_the_library_and_keeps_what_clients_know() {
        let dir = tempfile::tempdir().unwrap();
        let (layout, db) = legacy(dir.path()).await;
        let server_id = db.server_id().await.unwrap();
        assert!(refuse_start(&layout, 1).unwrap().contains("defter migrate"));
        // Its last writes are only in the write-ahead log until the checkpoint.
        let plan = plan(&layout, &owner()).unwrap();
        drop(db);
        assert_eq!(plan.version, 1);
        assert_eq!(plan.orcid.as_deref(), Some("0000-0003-1419-2405"));
        assert!(plan.profile_hidden);
        assert_eq!(plan.invites, ["0000-0002-1825-0097"]);
        let before = std::fs::read(&layout.legacy).unwrap().len();

        apply(&layout, &owner(), &plan).unwrap();
        assert_eq!(state(&layout, 1), State::Migrated);
        assert_eq!(refuse_start(&layout, 1), None);
        assert!(!layout.legacy.exists());
        let library = layout.library(LibraryRef::User(1));
        assert!(std::fs::read(&library).unwrap().len() >= before);

        let conn = control::open_connection(&layout.control).unwrap();
        let user = control::user(&conn, 1).unwrap().unwrap();
        assert!(user.legacy_files && user.profile_hidden);
        assert_eq!(user.orcid.as_deref(), Some("0000-0003-1419-2405"));
        assert_eq!(db::get_meta(&conn, "server_id").unwrap(), Some(server_id));
        assert_eq!(
            db::get_meta(&conn, "schema").unwrap().as_deref(),
            Some("{}")
        );
        assert_eq!(
            control::version_high(&conn, LibraryRef::User(1)).unwrap(),
            1
        );
        let invited: String = conn
            .query_row("SELECT orcid FROM invite", [], |r| r.get(0))
            .unwrap();
        assert_eq!(invited, "0000-0002-1825-0097");

        let moved = Db::open(&library).unwrap();
        let count: i64 = moved
            .read(|c| c.query_row("SELECT count(*) FROM object", [], |r| r.get(0)))
            .await
            .unwrap();
        assert_eq!(count, 1);
        assert!(plan_again_fails(&layout));
    }

    fn plan_again_fails(layout: &Layout) -> bool {
        plan(layout, &owner()).is_err_and(|e| e.contains("already migrated"))
    }

    #[tokio::test]
    async fn finishes_an_interrupted_migration() {
        let dir = tempfile::tempdir().unwrap();
        let (layout, db) = legacy(dir.path()).await;
        drop(db);
        let first = plan(&layout, &owner()).unwrap();
        create_control(&layout, &owner(), &first).unwrap();
        assert_eq!(state(&layout, 1), State::Interrupted);
        assert!(refuse_start(&layout, 1).unwrap().contains("again"));

        let again = plan(&layout, &owner()).unwrap();
        assert_eq!(again.state, State::Interrupted);
        apply(&layout, &owner(), &again).unwrap();
        assert_eq!(state(&layout, 1), State::Migrated);
        // Another owner cannot finish it.
        let dir = tempfile::tempdir().unwrap();
        let (layout, db) = legacy(dir.path()).await;
        drop(db);
        create_control(&layout, &owner(), &first).unwrap();
        let other = Owner {
            user_id: 2,
            ..owner()
        };
        let plan = plan(&layout, &other).unwrap();
        assert!(
            apply(&layout, &other, &plan)
                .unwrap_err()
                .contains("no user 2")
        );
    }

    #[tokio::test]
    async fn refuses_both_layouts_at_once() {
        let dir = tempfile::tempdir().unwrap();
        let (layout, db) = legacy(dir.path()).await;
        drop(db);
        let plan = plan(&layout, &owner()).unwrap();
        apply(&layout, &owner(), &plan).unwrap();
        // An older defter started afresh after the migration.
        drop(Db::open(&layout.legacy).unwrap());
        assert_eq!(state(&layout, 1), State::Both);
        assert!(refuse_start(&layout, 1).unwrap().contains("move whichever"));
        assert!(super::plan(&layout, &owner()).is_err());
    }

    #[tokio::test]
    async fn refuses_libraries_without_their_control_database() {
        let dir = tempfile::tempdir().unwrap();
        let (layout, db) = legacy(dir.path()).await;
        drop(db);
        let plan = plan(&layout, &owner()).unwrap();
        apply(&layout, &owner(), &plan).unwrap();
        std::fs::remove_file(&layout.control).unwrap();
        assert_eq!(state(&layout, 1), State::ControlMissing);
        assert!(
            refuse_start(&layout, 1)
                .unwrap()
                .contains("restore --control")
        );
        // Nor does a fresh library from an older defter count as one to migrate.
        drop(Db::open(&layout.legacy).unwrap());
        assert_eq!(state(&layout, 1), State::Both);
    }

    #[tokio::test]
    async fn never_moves_a_library_onto_another() {
        let dir = tempfile::tempdir().unwrap();
        let (layout, db) = legacy(dir.path()).await;
        drop(db);
        let plan = plan(&layout, &owner()).unwrap();
        create_control(&layout, &owner(), &plan).unwrap();
        std::fs::create_dir_all(&layout.libraries).unwrap();
        let stale = PathBuf::from(format!(
            "{}-wal",
            layout.library(LibraryRef::User(1)).display()
        ));
        std::fs::write(&stale, b"stale").unwrap();
        assert!(move_library(&layout, 1).unwrap_err().contains("in the way"));
        assert!(layout.legacy.exists());
    }

    #[test]
    fn a_new_server_needs_no_migration() {
        let dir = tempfile::tempdir().unwrap();
        let layout = Layout::new(dir.path(), dir.path().join("defter.sqlite3"));
        assert_eq!(state(&layout, 1), State::New);
        assert_eq!(refuse_start(&layout, 1), None);
        assert!(
            plan(&layout, &owner())
                .unwrap_err()
                .contains("nothing to migrate")
        );
    }
}