toko-feed-cli 0.3.4

//! Complete canonical JSON backup, excluding provider evidence and operations.

use std::{collections::BTreeSet, fs, path::Path};

use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use toko_feed::{
    CANONICAL_RESTORE_MAX_RECORDS, CanonicalItemSnapshot, CanonicalItemVariantSnapshot,
    CanonicalRestoreBatch, CanonicalRestoreReceipt, CanonicalSetSnapshot, CanonicalSnapshotArgs,
    CanonicalSnapshotKind, CanonicalSnapshotPage, CanonicalSnapshotRecords, CollectionView,
    ItemKind, MagicAbilitySnapshot, MagicCardFaceSnapshot, MagicCharacteristicSnapshot,
    MagicColorSnapshot, MagicFaceTypeSnapshot, MagicManaSymbolSnapshot,
    MagicMechanicOccurrenceSnapshot, MagicMechanicSnapshot, MagicTypeTermSnapshot,
    PokemonCardMetadataDetails,
};

use super::{
    CliError, DEFAULT_MAX_PAGES, ListOptions, MAX_QUERY_LIMIT, Target, call_one,
    collect_keyset_pages,
};

const CANONICAL_BACKUP_FORMAT_VERSION: u32 = 6;
const MAX_CANONICAL_BACKUP_BYTES: u64 = 64 * 1024 * 1024;
const POKEMON_METADATA_RESTORE_MAX_RECORDS: usize = 5;
const CANONICAL_SNAPSHOT_PAGE_RECORDS: u16 = MAX_QUERY_LIMIT;
pub const DEFAULT_CANONICAL_BACKUP_PATH: &str = "data/canonical.json";

#[derive(Debug, Deserialize, Serialize)]
struct CanonicalBackup {
    format_version: u32,
    collections: Vec<CollectionView>,
    sets: Vec<CanonicalSetSnapshot>,
    items: Vec<CanonicalItemSnapshot>,
    item_variants: Vec<CanonicalItemVariantSnapshot>,
    pokemon_card_metadata: Vec<PokemonCardMetadataDetails>,
    magic_type_terms: Vec<MagicTypeTermSnapshot>,
    magic_mechanics: Vec<MagicMechanicSnapshot>,
    magic_card_faces: Vec<MagicCardFaceSnapshot>,
    magic_characteristics: Vec<MagicCharacteristicSnapshot>,
    magic_mana_symbols: Vec<MagicManaSymbolSnapshot>,
    magic_colors: Vec<MagicColorSnapshot>,
    magic_face_types: Vec<MagicFaceTypeSnapshot>,
    magic_abilities: Vec<MagicAbilitySnapshot>,
    magic_mechanic_occurrences: Vec<MagicMechanicOccurrenceSnapshot>,
}

pub fn execute(target: &Target, path: &Path) -> Result<Value, CliError> {
    let (backup, pages) = capture(target)?;
    write_backup(path, &backup)?;
    Ok(json!({
        "path": path,
        "format_version": CANONICAL_BACKUP_FORMAT_VERSION,
        "collections": backup.collections.len(),
        "sets": backup.sets.len(),
        "items": backup.items.len(),
        "item_variants": backup.item_variants.len(),
        "pokemon_card_metadata": backup.pokemon_card_metadata.len(),
        "magic_card_faces": backup.magic_card_faces.len(),
        "magic_extension_rows": magic_extension_rows(&backup),
        "pages": pages,
    }))
}

fn capture(target: &Target) -> Result<(CanonicalBackup, usize), CliError> {
    let options = ListOptions {
        limit: CANONICAL_SNAPSHOT_PAGE_RECORDS,
        after: None,
        all: true,
        max_pages: DEFAULT_MAX_PAGES,
    };

    macro_rules! collect_snapshot {
        ($kind:ident, $variant:ident) => {{
            collect_keyset_pages(&options, |after, limit| {
                let page = fetch_snapshot_page(target, CanonicalSnapshotKind::$kind, after, limit)?;
                let CanonicalSnapshotRecords::$variant(records) = page.records else {
                    return Err(CliError::Usage(
                        "canister returned the wrong canonical snapshot row type".to_owned(),
                    ));
                };
                Ok((records, page.next_after))
            })?
        }};
    }

    let (collections, collection_pages) = collect_snapshot!(Collections, Collections);
    let (sets, set_pages) = collect_snapshot!(Sets, Sets);
    let (items, item_pages) = collect_snapshot!(Items, Items);
    let (item_variants, item_variant_pages) = collect_snapshot!(ItemVariants, ItemVariants);
    let (magic_type_terms, magic_type_term_pages) =
        collect_snapshot!(MagicTypeTerms, MagicTypeTerms);
    let (magic_mechanics, magic_mechanic_pages) = collect_snapshot!(MagicMechanics, MagicMechanics);
    let (magic_card_faces, magic_card_face_pages) =
        collect_snapshot!(MagicCardFaces, MagicCardFaces);
    let (magic_characteristics, magic_characteristic_pages) =
        collect_snapshot!(MagicCharacteristics, MagicCharacteristics);
    let (magic_mana_symbols, magic_mana_symbol_pages) =
        collect_snapshot!(MagicManaSymbols, MagicManaSymbols);
    let (magic_colors, magic_color_pages) = collect_snapshot!(MagicColors, MagicColors);
    let (magic_face_types, magic_face_type_pages) =
        collect_snapshot!(MagicFaceTypes, MagicFaceTypes);
    let (magic_abilities, magic_ability_pages) = collect_snapshot!(MagicAbilities, MagicAbilities);
    let (magic_mechanic_occurrences, magic_occurrence_pages) =
        collect_snapshot!(MagicMechanicOccurrences, MagicMechanicOccurrences);

    let pokemon_collection_id = collections
        .iter()
        .find(|collection| collection.name == "Pokemon")
        .map(|collection| collection.id.as_str());
    let pokemon_set_ids = sets
        .iter()
        .filter(|set| Some(set.collection_id.as_str()) == pokemon_collection_id)
        .map(|set| set.id.as_str())
        .collect::<BTreeSet<_>>();
    let mut pokemon_card_metadata = items
        .iter()
        .filter(|item| {
            item.kind == ItemKind::Card && pokemon_set_ids.contains(item.set_id.as_str())
        })
        .map(|item| {
            call_one::<_, Option<PokemonCardMetadataDetails>>(
                target,
                "toko_feed_card_metadata",
                item.id.clone(),
                true,
            )
        })
        .collect::<Result<Vec<_>, CliError>>()?
        .into_iter()
        .flatten()
        .collect::<Vec<_>>();
    for details in &mut pokemon_card_metadata {
        details.evidence.clear();
    }

    let pages = collection_pages
        .saturating_add(set_pages)
        .saturating_add(item_pages)
        .saturating_add(item_variant_pages)
        .saturating_add(magic_type_term_pages)
        .saturating_add(magic_mechanic_pages)
        .saturating_add(magic_card_face_pages)
        .saturating_add(magic_characteristic_pages)
        .saturating_add(magic_mana_symbol_pages)
        .saturating_add(magic_color_pages)
        .saturating_add(magic_face_type_pages)
        .saturating_add(magic_ability_pages)
        .saturating_add(magic_occurrence_pages);
    Ok((
        CanonicalBackup {
            format_version: CANONICAL_BACKUP_FORMAT_VERSION,
            collections,
            sets,
            items,
            item_variants,
            pokemon_card_metadata,
            magic_type_terms,
            magic_mechanics,
            magic_card_faces,
            magic_characteristics,
            magic_mana_symbols,
            magic_colors,
            magic_face_types,
            magic_abilities,
            magic_mechanic_occurrences,
        },
        pages,
    ))
}

pub fn restore(target: &Target, path: &Path) -> Result<Value, CliError> {
    let mut backup = read_backup(path)?;
    for details in &mut backup.pokemon_card_metadata {
        details.evidence.clear();
    }
    let receipts = restore_all(target, &backup)?;
    Ok(json!({
        "path": path,
        "format_version": backup.format_version,
        "batches": receipts.len(),
        "submitted": receipts.iter().map(|receipt| u64::from(receipt.submitted)).sum::<u64>(),
        "inserted": receipts.iter().map(|receipt| u64::from(receipt.inserted)).sum::<u64>(),
        "unchanged": receipts.iter().map(|receipt| u64::from(receipt.unchanged)).sum::<u64>(),
        "receipts": receipts,
    }))
}

fn read_backup(path: &Path) -> Result<CanonicalBackup, CliError> {
    let metadata = fs::metadata(path).map_err(|source| CliError::BackupIo {
        action: "read metadata for",
        path: path.to_owned(),
        source,
    })?;
    if metadata.len() > MAX_CANONICAL_BACKUP_BYTES {
        return Err(CliError::Usage(format!(
            "canonical backup exceeds the {} MiB input limit",
            MAX_CANONICAL_BACKUP_BYTES / 1024 / 1024
        )));
    }
    let bytes = fs::read(path).map_err(|source| CliError::BackupIo {
        action: "read",
        path: path.to_owned(),
        source,
    })?;
    let backup = serde_json::from_slice::<CanonicalBackup>(&bytes)?;
    if backup.format_version != CANONICAL_BACKUP_FORMAT_VERSION {
        return Err(CliError::Usage(format!(
            "unsupported canonical backup format {}; expected {}",
            backup.format_version, CANONICAL_BACKUP_FORMAT_VERSION
        )));
    }
    Ok(backup)
}

fn restore_all(
    target: &Target,
    backup: &CanonicalBackup,
) -> Result<Vec<CanonicalRestoreReceipt>, CliError> {
    let mut receipts = Vec::new();
    restore_chunks(
        target,
        &backup.collections,
        CanonicalRestoreBatch::Collections,
        &mut receipts,
    )?;
    restore_chunks(
        target,
        &backup.sets,
        CanonicalRestoreBatch::Sets,
        &mut receipts,
    )?;
    restore_chunks(
        target,
        &backup.items,
        CanonicalRestoreBatch::Items,
        &mut receipts,
    )?;
    restore_chunks(
        target,
        &backup.item_variants,
        CanonicalRestoreBatch::ItemVariants,
        &mut receipts,
    )?;
    restore_chunks_at_most(
        target,
        &backup.pokemon_card_metadata,
        POKEMON_METADATA_RESTORE_MAX_RECORDS,
        CanonicalRestoreBatch::PokemonCardMetadata,
        &mut receipts,
    )?;
    restore_chunks(
        target,
        &backup.magic_type_terms,
        CanonicalRestoreBatch::MagicTypeTerms,
        &mut receipts,
    )?;
    restore_chunks(
        target,
        &backup.magic_mechanics,
        CanonicalRestoreBatch::MagicMechanics,
        &mut receipts,
    )?;
    restore_chunks(
        target,
        &backup.magic_card_faces,
        CanonicalRestoreBatch::MagicCardFaces,
        &mut receipts,
    )?;
    restore_chunks(
        target,
        &backup.magic_characteristics,
        CanonicalRestoreBatch::MagicCharacteristics,
        &mut receipts,
    )?;
    restore_chunks(
        target,
        &backup.magic_mana_symbols,
        CanonicalRestoreBatch::MagicManaSymbols,
        &mut receipts,
    )?;
    restore_chunks(
        target,
        &backup.magic_colors,
        CanonicalRestoreBatch::MagicColors,
        &mut receipts,
    )?;
    restore_chunks(
        target,
        &backup.magic_face_types,
        CanonicalRestoreBatch::MagicFaceTypes,
        &mut receipts,
    )?;
    restore_chunks(
        target,
        &backup.magic_abilities,
        CanonicalRestoreBatch::MagicAbilities,
        &mut receipts,
    )?;
    restore_chunks(
        target,
        &backup.magic_mechanic_occurrences,
        CanonicalRestoreBatch::MagicMechanicOccurrences,
        &mut receipts,
    )?;
    Ok(receipts)
}

fn fetch_snapshot_page(
    target: &Target,
    kind: CanonicalSnapshotKind,
    after: Option<String>,
    limit: u16,
) -> Result<CanonicalSnapshotPage, CliError> {
    call_one(
        target,
        "toko_feed_canonical_snapshot",
        CanonicalSnapshotArgs { kind, after, limit },
        true,
    )
}

fn restore_chunks<T, F>(
    target: &Target,
    records: &[T],
    batch: F,
    receipts: &mut Vec<CanonicalRestoreReceipt>,
) -> Result<(), CliError>
where
    T: Clone,
    F: Fn(Vec<T>) -> CanonicalRestoreBatch,
{
    restore_chunks_at_most(
        target,
        records,
        CANONICAL_RESTORE_MAX_RECORDS,
        batch,
        receipts,
    )
}

fn restore_chunks_at_most<T, F>(
    target: &Target,
    records: &[T],
    maximum: usize,
    batch: F,
    receipts: &mut Vec<CanonicalRestoreReceipt>,
) -> Result<(), CliError>
where
    T: Clone,
    F: Fn(Vec<T>) -> CanonicalRestoreBatch,
{
    for chunk in records.chunks(maximum) {
        receipts.push(call_one(
            target,
            "toko_feed_restore_canonical",
            batch(chunk.to_vec()),
            false,
        )?);
    }
    Ok(())
}

const fn magic_extension_rows(backup: &CanonicalBackup) -> usize {
    backup
        .magic_type_terms
        .len()
        .saturating_add(backup.magic_mechanics.len())
        .saturating_add(backup.magic_card_faces.len())
        .saturating_add(backup.magic_characteristics.len())
        .saturating_add(backup.magic_mana_symbols.len())
        .saturating_add(backup.magic_colors.len())
        .saturating_add(backup.magic_face_types.len())
        .saturating_add(backup.magic_abilities.len())
        .saturating_add(backup.magic_mechanic_occurrences.len())
}

fn write_backup(path: &Path, backup: &CanonicalBackup) -> Result<(), CliError> {
    let parent = path
        .parent()
        .filter(|parent| !parent.as_os_str().is_empty());
    if let Some(parent) = parent {
        fs::create_dir_all(parent).map_err(|source| CliError::BackupIo {
            action: "create parent for",
            path: parent.to_owned(),
            source,
        })?;
    }
    let file_name = path
        .file_name()
        .and_then(|name| name.to_str())
        .ok_or_else(|| {
            CliError::Usage("canonical backup path must name a UTF-8 JSON file".to_owned())
        })?;
    let temporary = path.with_file_name(format!(".{file_name}.{}.tmp", std::process::id()));
    let mut bytes = serde_json::to_vec_pretty(backup)?;
    bytes.push(b'\n');
    fs::write(&temporary, bytes).map_err(|source| CliError::BackupIo {
        action: "write",
        path: temporary.clone(),
        source,
    })?;
    if let Err(source) = fs::rename(&temporary, path) {
        let _ = fs::remove_file(&temporary);
        return Err(CliError::BackupIo {
            action: "replace",
            path: path.to_owned(),
            source,
        });
    }
    Ok(())
}