oxyromon 0.24.0

use super::common::*;
use super::database::*;
use super::import_roms::{UnattendedMode, import_rom};
use super::mimetype::*;
use super::model::*;
use super::progress::*;
use super::util::*;
use anyhow::{Context, Result, anyhow};
use clap::value_parser;
use clap::{Arg, ArgAction, ArgMatches, Command};
use console::Style;
use indicatif::ProgressBar;
use quick_xml::de;
use rayon::prelude::*;
use rust_embed::RustEmbed;
use serde::Serialize;
use shiratsu_naming::naming::TokenizedName;
use shiratsu_naming::naming::nointro::{NoIntroName, NoIntroToken};
use shiratsu_naming::naming::tosec::{TOSECName, TOSECToken};
use shiratsu_naming::region::Region;
use sqlx::sqlite::SqliteConnection;
use std::io;
use std::path::Path;
use std::path::PathBuf;
use std::str;
use vec_drain_where::VecDrainWhereExt;
use walkdir::WalkDir;
use zip::ZipArchive;

#[derive(RustEmbed)]
#[folder = "data/"]
struct Assets;

#[derive(Clone, Debug, Serialize)]
pub struct ImportSummary {
    pub system_name: String,
    pub system_version: String,
    pub game_count: usize,
}

pub fn subcommand() -> Command {
    Command::new("import-dats")
        .about("Parse and import Logiqx DAT files into oxyromon")
        .arg(
            Arg::new("DATS")
                .help("Set the DAT files to import")
                .required(true)
                .num_args(1..)
                .index(1)
                .value_parser(value_parser!(PathBuf)),
        )
        .arg(
            Arg::new("INFO")
                .short('i')
                .long("info")
                .help("Show the DAT information and exit")
                .required(false)
                .action(ArgAction::SetTrue),
        )
        .arg(
            Arg::new("UPDATE")
                .short('u')
                .long("update")
                .help("Only import DAT files for systems already in the database")
                .required(false)
                .action(ArgAction::SetTrue),
        )
        .arg(
            Arg::new("SKIP_HEADER")
                .short('s')
                .long("skip-header")
                .help("Skip parsing the header even if the system has one")
                .required(false)
                .action(ArgAction::SetTrue),
        )
        .arg(
            Arg::new("FORCE")
                .short('f')
                .long("force")
                .help("Force import of outdated DAT files")
                .required(false)
                .action(ArgAction::SetTrue),
        )
        .arg(
            Arg::new("NAME")
                .short('n')
                .long("name")
                .help("Customize the system name")
                .required(false)
                .num_args(1)
                .conflicts_with("UPDATE"),
        )
        .arg(
            Arg::new("EXTENSION")
                .short('e')
                .long("extension")
                .help("Customize the system extension")
                .required(false)
                .num_args(1)
                .conflicts_with("UPDATE"),
        )
}

pub async fn main(
    connection: &mut SqliteConnection,
    matches: &ArgMatches,
    progress_bar: &ProgressBar,
) -> Result<()> {
    let (zip_paths, mut dat_paths): (Vec<PathBuf>, Vec<PathBuf>) = matches
        .get_many::<PathBuf>("DATS")
        .unwrap()
        .cloned()
        .partition(|path| {
            path.extension()
                .map(|ext| ext.to_str().unwrap().to_lowercase() == ZIP_EXTENSION)
                .unwrap_or(false)
        });

    let tmp_directory = create_tmp_directory(connection).await?;
    for zip_path in zip_paths {
        let mut reader = get_reader_sync(&zip_path)?;
        let mut zip_archive = ZipArchive::new(&mut reader).context("Failed to read ZIP")?;
        zip_archive
            .extract(&tmp_directory)
            .context("Failed to extract ZIP")?;
        // walk the extracted tree to find all .dat files, including in subdirectories
        let walker = WalkDir::new(tmp_directory.path()).into_iter();
        for entry in walker.filter_map(|e| e.ok()) {
            if entry.path().is_file()
                && entry
                    .path()
                    .extension()
                    .is_some_and(|ext| ext.to_str().unwrap().to_lowercase() == DAT_EXTENSION)
            {
                dat_paths.push(entry.path().to_path_buf());
            }
        }
    }

    // deduplicate dat paths that may have been found multiple times
    dat_paths.sort();
    dat_paths.dedup();

    let update = matches.get_flag("UPDATE");

    let custom_name = matches.get_one::<String>("NAME");
    if let Some(custom_name) = custom_name {
        if dat_paths.len() > 1 {
            print_error(
                progress_bar,
                "Custom system name requires a single DAT file",
            );
            return Ok(());
        }
        if find_system_by_name(connection, custom_name).await.is_some() {
            print_error(
                progress_bar,
                "Custom system name must not match a known system name",
            );
            return Ok(());
        }
    }

    let custom_extension = matches.get_one::<String>("EXTENSION");
    if custom_extension.is_some() && dat_paths.len() > 1 {
        print_error(
            progress_bar,
            "Custom system extension requires a single DAT file",
        );
        return Ok(());
    }

    for dat_path in dat_paths {
        print_header(
            progress_bar,
            &format!(
                "Processing \"{}\"",
                dat_path.file_name().unwrap().to_str().unwrap()
            ),
        );
        let (datfile_xml, detector_xml) = parse_dat(
            progress_bar,
            &get_canonicalized_path(&dat_path).await?,
            matches.get_flag("SKIP_HEADER"),
        )
        .await?;

        // in update mode, skip systems that are not already in the database
        if update {
            let system_name = &datfile_xml.system.name;
            if find_system_by_name(connection, system_name).await.is_none() {
                print_skip(
                    progress_bar,
                    &format!("Skipping unknown system \"{}\"", system_name),
                );
                print_separator(progress_bar);
                continue;
            }
        }

        if custom_extension.is_some() {
            let games = if !datfile_xml.machines.is_empty() {
                &datfile_xml.machines
            } else {
                &datfile_xml.games
            };

            for game in games {
                if game.roms.len() + game.disks.len() > 1 {
                    print_error(
                        progress_bar,
                        "Custom extension cannot be used with games that have multiple ROMs",
                    );
                    return Ok(());
                }
            }
        }

        if !matches.get_flag("INFO") {
            import_dat(
                connection,
                progress_bar,
                &datfile_xml,
                &detector_xml,
                custom_name,
                custom_extension,
                matches.get_flag("FORCE"),
            )
            .await?;
        }
        print_separator(progress_bar);
    }
    Ok(())
}

pub async fn parse_dat<P: AsRef<Path>>(
    progress_bar: &ProgressBar,
    dat_path: &P,
    skip_header: bool,
) -> Result<(DatfileXml, Option<DetectorXml>)> {
    let datfile_xml: DatfileXml = de::from_reader(&mut get_reader_sync(dat_path)?)
        .context("Failed to deserialize DAT file")?;

    // print information
    let label_style = Style::new().dim();
    print_info(
        progress_bar,
        &format!(
            "{} {}",
            label_style.apply_to("System:"),
            datfile_xml.system.name,
        ),
    );
    print_info(
        progress_bar,
        &format!(
            "{} {}",
            label_style.apply_to("Version:"),
            datfile_xml.system.version,
        ),
    );
    if !datfile_xml.machines.is_empty() {
        print_info(
            progress_bar,
            &format!(
                "{} {}",
                label_style.apply_to("Games:"),
                datfile_xml.machines.len(),
            ),
        );
    } else {
        print_info(
            progress_bar,
            &format!(
                "{} {}",
                label_style.apply_to("Games:"),
                datfile_xml.games.len(),
            ),
        );
    }

    let mut detector_xml = None;
    if !skip_header
        && let Some(clr_mame_pro_xml) = &datfile_xml
            .system
            .clrmamepros
            .iter()
            .find(|clrmamepro| clrmamepro.header.is_some())
    {
        print_subheader(progress_bar, "Processing header");
        if let Some(header_file_name) = &clr_mame_pro_xml.header {
            let header_file_path = dat_path.as_ref().parent().unwrap().join(header_file_name);
            if header_file_path.is_file() {
                let header_file = open_file_sync(&header_file_path.as_path())?;
                let reader = io::BufReader::new(header_file);
                detector_xml = de::from_reader(reader).expect("Failed to parse header file");
            } else {
                let header_file = Assets::get(header_file_name).unwrap();
                detector_xml = de::from_str(str::from_utf8(header_file.data.as_ref()).unwrap())
                    .expect("Failed to parse header file");
            }
        }
    };

    Ok((datfile_xml, detector_xml))
}

pub async fn import_dat(
    connection: &mut SqliteConnection,
    progress_bar: &ProgressBar,
    datfile_xml: &DatfileXml,
    detector_xml: &Option<DetectorXml>,
    custom_name: Option<&String>,
    custom_extension: Option<&String>,
    force: bool,
) -> Result<Option<ImportSummary>> {
    print_subheader(progress_bar, "Processing system");

    let game_count = if !datfile_xml.machines.is_empty() {
        datfile_xml.machines.len()
    } else {
        datfile_xml.games.len()
    };

    let mut transaction = begin_transaction(connection).await;

    // persist system
    let system_id = match create_or_update_system(
        &mut transaction,
        progress_bar,
        &datfile_xml.system,
        custom_name,
        custom_extension,
        !datfile_xml.machines.is_empty(),
        force,
    )
    .await
    {
        Some(system_id) => system_id,
        None => return Ok(None),
    };

    // persist header
    if let Some(detector_xml) = detector_xml {
        create_or_update_header(&mut transaction, detector_xml, system_id).await;
    }

    // persist games
    progress_bar.set_style(get_count_progress_style());
    if !datfile_xml.machines.is_empty() {
        progress_bar.set_length(datfile_xml.machines.len() as u64);
    } else {
        progress_bar.set_length(datfile_xml.games.len() as u64);
    }

    let mut orphan_romfile_ids: Vec<i64> = vec![];
    print_subheader(progress_bar, "Deleting old games");
    orphan_romfile_ids.append(
        &mut delete_old_games(
            &mut transaction,
            if !datfile_xml.machines.is_empty() {
                &datfile_xml.machines
            } else {
                &datfile_xml.games
            },
            system_id,
        )
        .await,
    );
    print_subheader(progress_bar, "Processing games");
    orphan_romfile_ids.append(
        &mut create_or_update_games(
            &mut transaction,
            if !datfile_xml.machines.is_empty() {
                &datfile_xml.machines
            } else {
                &datfile_xml.games
            },
            system_id,
            !datfile_xml.machines.is_empty(),
            progress_bar,
        )
        .await?,
    );

    progress_bar.set_style(get_none_progress_style());
    progress_bar.reset();

    // reimport orphan romfiles
    if !orphan_romfile_ids.is_empty() {
        print_subheader(progress_bar, "Processing orphan romfiles");
        orphan_romfile_ids.dedup();
        reimport_orphan_romfiles(
            &mut transaction,
            progress_bar,
            system_id,
            orphan_romfile_ids,
        )
        .await?;
    }

    // create necessary directories
    let system = find_system_by_id(&mut transaction, system_id).await;
    get_system_directory(&mut transaction, &system).await?;
    get_trash_directory(&mut transaction, Some(&system)).await?;

    // update games and systems completion
    compute_system_completion(&mut transaction, progress_bar, &system).await?;

    commit_transaction(transaction).await;

    Ok(Some(ImportSummary {
        system_name: datfile_xml.system.name.clone(),
        system_version: datfile_xml.system.version.clone(),
        game_count,
    }))
}

fn get_regions_from_game_name(name: &str) -> Result<String> {
    match NoIntroName::try_parse(name) {
        Ok(v) => {
            for token in v.iter() {
                if let NoIntroToken::Region {
                    region_strings: _,
                    regions,
                } = token
                {
                    return Ok(Region::to_normalized_region_string(regions));
                }
            }
        }
        Err(_) => match TOSECName::try_parse(name) {
            Ok(v) => {
                for token in v.iter() {
                    if let TOSECToken::Region {
                        region_strings: _,
                        regions,
                    } = token
                    {
                        return Ok(Region::to_normalized_region_string(regions));
                    }
                }
            }
            Err(e) => {
                return Err(anyhow!("Failed to parse name: {}", e));
            }
        },
    };
    Ok(String::from(""))
}

async fn create_or_update_system(
    connection: &mut SqliteConnection,
    progress_bar: &ProgressBar,
    system_xml: &SystemXml,
    custom_name: Option<&String>,
    custom_extension: Option<&String>,
    arcade: bool,
    force: bool,
) -> Option<i64> {
    match find_system_by_name(connection, &system_xml.name).await {
        Some(system) => {
            if is_update(progress_bar, &system.version, &system_xml.version) || force {
                update_system_from_xml(
                    connection,
                    system.id,
                    system_xml,
                    custom_name,
                    custom_extension,
                    arcade,
                )
                .await;
                Some(system.id)
            } else {
                None
            }
        }
        None => Some(
            create_system_from_xml(
                connection,
                system_xml,
                custom_name,
                custom_extension,
                arcade,
            )
            .await,
        ),
    }
}

async fn create_or_update_header(
    connection: &mut SqliteConnection,
    detector_xml: &DetectorXml,
    system_id: i64,
) {
    let header_id = match find_header_by_system_id(connection, system_id).await {
        Some(header) => {
            update_header_from_xml(connection, header.id, detector_xml, system_id).await;
            delete_rules_by_header_id(connection, header.id).await;
            header.id
        }
        None => create_header_from_xml(connection, detector_xml, system_id).await,
    };
    for data_xml in &detector_xml.rule.data {
        create_rule_from_xml(connection, data_xml, header_id).await;
    }
}

async fn create_or_update_games(
    connection: &mut SqliteConnection,
    games_xml: &[GameXml],
    system_id: i64,
    arcade: bool,
    progress_bar: &ProgressBar,
) -> Result<Vec<i64>> {
    let mut orphan_romfile_ids: Vec<i64> = vec![];
    let (mut parent_games_xml, mut child_games_xml): (Vec<&GameXml>, Vec<&GameXml>) = games_xml
        .par_iter()
        .partition(|game_xml| game_xml.cloneof.is_none() && game_xml.romof.is_none());
    for game_xml in &parent_games_xml {
        let game = find_game_by_name_and_bios_and_system_id(
            connection,
            &game_xml.name,
            game_xml.isbios,
            system_id,
        )
        .await;
        let mut regions = String::new();
        if !arcade {
            match get_regions_from_game_name(&game_xml.name) {
                Ok(s) => regions.push_str(&s),
                Err(err) => {
                    print_warning(progress_bar, &format!("{:#}", err));
                    progress_bar.inc(1);
                    continue;
                }
            }
        }
        let game_id = match game {
            Some(game) => {
                update_game_from_xml(
                    connection, game.id, game_xml, &regions, system_id, None, None,
                )
                .await;
                game.id
            }
            None => {
                create_game_from_xml(connection, game_xml, &regions, system_id, None, None).await
            }
        };
        if !game_xml.roms.is_empty() {
            orphan_romfile_ids.append(
                &mut create_or_update_roms(
                    connection,
                    progress_bar,
                    &game_xml.roms,
                    game_xml.isbios,
                    false,
                    game_id,
                )
                .await,
            );
        }
        if !game_xml.disks.is_empty() {
            orphan_romfile_ids.append(
                &mut create_or_update_roms(
                    connection,
                    progress_bar,
                    &game_xml.disks,
                    game_xml.isbios,
                    true,
                    game_id,
                )
                .await,
            );
        }
        orphan_romfile_ids.append(
            &mut delete_old_roms(
                connection,
                &[
                    game_xml.roms.iter().collect::<Vec<&RomXml>>(),
                    game_xml.disks.iter().collect::<Vec<&RomXml>>(),
                ]
                .concat(),
                game_id,
            )
            .await,
        );
        progress_bar.inc(1)
    }
    while !child_games_xml.is_empty() {
        let parent_game_names: Vec<&str> = parent_games_xml
            .par_iter()
            .map(|game_xml| game_xml.name.as_str())
            .collect();
        parent_games_xml = child_games_xml
            .e_drain_where(|&mut child_game_xml| {
                parent_game_names.contains(
                    &child_game_xml
                        .cloneof
                        .as_ref()
                        .or(child_game_xml.romof.as_ref())
                        .unwrap()
                        .as_str(),
                )
            })
            .collect();
        if parent_games_xml.is_empty() {
            for child_game_xml in &child_games_xml {
                print_warning(
                    progress_bar,
                    &format!(
                        "Game \"{}\" has an invalid parent: \"{}\"",
                        child_game_xml.name,
                        child_game_xml
                            .cloneof
                            .as_ref()
                            .or(child_game_xml.romof.as_ref())
                            .unwrap()
                            .as_str(),
                    ),
                );
            }
            break;
        }
        for game_xml in &parent_games_xml {
            let game = find_game_by_name_and_bios_and_system_id(
                connection,
                &game_xml.name,
                game_xml.isbios,
                system_id,
            )
            .await;
            // sometimes romof refers to games that aren't bioses
            let parent_game = match game_xml.cloneof.as_ref() {
                Some(name) => {
                    find_game_by_name_and_bios_and_system_id(connection, name, false, system_id)
                        .await
                }
                None => match game_xml.romof.as_ref() {
                    Some(name) => {
                        find_game_by_name_and_bios_and_system_id(connection, name, false, system_id)
                            .await
                    }
                    None => None,
                },
            };
            let bios_game: Option<Game> = match game_xml.romof.as_ref() {
                Some(name) => {
                    find_game_by_name_and_bios_and_system_id(connection, name, true, system_id)
                        .await
                }
                None => None,
            };
            let mut regions = String::new();
            if !arcade {
                match get_regions_from_game_name(&game_xml.name) {
                    Ok(s) => regions.push_str(&s),
                    Err(err) => {
                        print_warning(progress_bar, &format!("{:#}", err));
                        progress_bar.inc(1);
                        continue;
                    }
                }
            }
            let game_id = match game {
                Some(game) => {
                    update_game_from_xml(
                        connection,
                        game.id,
                        game_xml,
                        &regions,
                        system_id,
                        parent_game.map(|game| game.id),
                        bios_game.map(|game| game.id),
                    )
                    .await;
                    game.id
                }
                None => {
                    create_game_from_xml(
                        connection,
                        game_xml,
                        &regions,
                        system_id,
                        parent_game.map(|game| game.id),
                        bios_game.map(|game| game.id),
                    )
                    .await
                }
            };
            if !game_xml.roms.is_empty() {
                orphan_romfile_ids.append(
                    &mut create_or_update_roms(
                        connection,
                        progress_bar,
                        &game_xml.roms,
                        game_xml.isbios,
                        false,
                        game_id,
                    )
                    .await,
                );
            }
            if !game_xml.disks.is_empty() {
                orphan_romfile_ids.append(
                    &mut create_or_update_roms(
                        connection,
                        progress_bar,
                        &game_xml.disks,
                        game_xml.isbios,
                        true,
                        game_id,
                    )
                    .await,
                );
            }
            orphan_romfile_ids.append(
                &mut delete_old_roms(
                    connection,
                    &[
                        game_xml.roms.iter().collect::<Vec<&RomXml>>(),
                        game_xml.disks.iter().collect::<Vec<&RomXml>>(),
                    ]
                    .concat(),
                    game_id,
                )
                .await,
            );
            progress_bar.inc(1)
        }
    }
    Ok(orphan_romfile_ids)
}

async fn create_or_update_roms(
    connection: &mut SqliteConnection,
    progress_bar: &ProgressBar,
    roms_xml: &[RomXml],
    bios: bool,
    disk: bool,
    game_id: i64,
) -> Vec<i64> {
    let mut orphan_romfile_ids: Vec<i64> = vec![];
    for rom_xml in roms_xml {
        // skip nodump roms
        if rom_xml.status.as_deref() == Some("nodump") {
            continue;
        }
        // find parent rom if needed
        let mut rom_parent_bios = false;
        let mut rom_parent_id = None;
        if rom_xml.merge.is_some()
            && let Some(crc) = &rom_xml.crc
        {
            let game = find_game_by_id(connection, game_id).await;
            let mut game_ids: Vec<i64> = vec![];
            if let Some(bios_id) = game.bios_id {
                game_ids.push(bios_id);
            }
            if let Some(parent_id) = game.parent_id {
                game_ids.push(parent_id);
                let parent_game = find_game_by_id(connection, parent_id).await;
                if let Some(bios_id) = parent_game.bios_id {
                    game_ids.push(bios_id);
                }
            }
            let roms =
                find_rom_by_size_and_crc_and_game_ids(connection, rom_xml.size, crc, &game_ids)
                    .await;
            if let Some(rom) = roms.first() {
                rom_parent_bios = rom.bios;
                rom_parent_id = Some(rom.id);
            } else {
                print_warning(
                    progress_bar,
                    &format!(
                        "ROM \"{}\" not found in game \"{}\" parent/bios, please fix your DAT file",
                        rom_xml.name, game.name
                    ),
                );
            }
        }
        match find_rom_by_name_and_game_id(connection, &rom_xml.name, game_id).await {
            Some(rom) => {
                update_rom_from_xml(
                    connection,
                    rom.id,
                    rom_xml,
                    bios || rom_parent_bios,
                    disk,
                    game_id,
                    rom_parent_id,
                    PathBuf::from(&rom_xml.name)
                        .extension()
                        .is_some_and(|extension| {
                            !NON_ORIGINAL_EXTENSIONS.contains(&extension.to_str().unwrap())
                        }),
                )
                .await;
                if (rom_xml.size != rom.size
                    || rom_xml.crc.is_some()
                        && rom_xml.crc.as_ref().unwrap() != rom.crc.as_ref().unwrap())
                    && let Some(romfile_id) = rom.romfile_id
                {
                    orphan_romfile_ids.push(romfile_id);
                    update_rom_romfile(connection, rom.id, None).await;
                }
                rom.id
            }
            None => {
                create_rom_from_xml(
                    connection,
                    rom_xml,
                    bios || rom_parent_bios,
                    disk,
                    game_id,
                    rom_parent_id,
                    PathBuf::from(&rom_xml.name)
                        .extension()
                        .is_some_and(|extension| {
                            !NON_ORIGINAL_EXTENSIONS.contains(&extension.to_str().unwrap())
                        }),
                )
                .await
            }
        };
    }
    orphan_romfile_ids
}

async fn delete_old_games(
    connection: &mut SqliteConnection,
    games_xml: &[GameXml],
    system_id: i64,
) -> Vec<i64> {
    let mut orphan_romfile_ids: Vec<i64> = vec![];
    let game_names_xml: Vec<&String> = games_xml.iter().map(|game_xml| &game_xml.name).collect();
    let games: Vec<Game> = find_games_by_system_id(connection, system_id)
        .await
        .into_par_iter()
        .filter(|game| !game_names_xml.contains(&&game.name))
        .collect();
    for game in games {
        orphan_romfile_ids.extend(
            find_roms_by_game_id_no_parents(connection, game.id)
                .await
                .into_iter()
                .filter_map(|rom| rom.romfile_id),
        );
        delete_game_by_name_and_system_id(connection, &game.name, system_id).await;
    }
    orphan_romfile_ids
}

async fn delete_old_roms(
    connection: &mut SqliteConnection,
    roms_xml: &[&RomXml],
    game_id: i64,
) -> Vec<i64> {
    let rom_names_romfile_ids: Vec<(String, Option<i64>)> =
        find_roms_by_game_id_no_parents(connection, game_id)
            .await
            .into_par_iter()
            .map(|rom| (rom.name, rom.romfile_id))
            .collect();
    let mut orphan_romfile_ids: Vec<i64> = vec![];
    for (rom_name, rom_romfile_id) in &rom_names_romfile_ids {
        if !roms_xml.iter().any(|rom_xml| &rom_xml.name == rom_name) {
            delete_rom_by_name_and_game_id(connection, rom_name, game_id).await;
            if let Some(romfile_id) = rom_romfile_id {
                orphan_romfile_ids.push(*romfile_id);
            }
        }
    }
    orphan_romfile_ids
}

pub async fn reimport_orphan_romfiles(
    connection: &mut SqliteConnection,
    progress_bar: &ProgressBar,
    system_id: i64,
    orphan_romfile_ids: Vec<i64>,
) -> Result<()> {
    let system = find_system_by_id(connection, system_id).await;
    let header = find_header_by_system_id(connection, system_id).await;
    for romfile_id in orphan_romfile_ids {
        let romfile = find_romfile_by_id(connection, romfile_id)
            .await
            .as_common(connection)
            .await?;
        delete_romfile_by_id(connection, romfile_id).await;
        if romfile.path.is_file() {
            let (_, game_ids) = import_rom(
                connection,
                progress_bar,
                &Some(&system),
                &header,
                &romfile.path,
                false,
                false,
                false,
                UnattendedMode::Skip,
                false,
            )
            .await?;
            if game_ids.is_empty() {
                let new_path = get_trash_directory(connection, Some(&system))
                    .await?
                    .join(romfile.path.file_name().unwrap().to_str().unwrap());
                romfile
                    .rename(progress_bar, &new_path, false)
                    .await?
                    .create(connection, progress_bar, RomfileType::Romfile)
                    .await?;
            }
        }
    }
    Ok(())
}

#[cfg(feature = "server")]
pub enum ImportDatResult {
    Imported(ImportSummary),
    UpToDate(String),
    Skipped,
}

#[cfg(feature = "server")]
pub async fn process_dat_upload(
    connection: &mut SqliteConnection,
    progress_bar: &ProgressBar,
    file_path: &PathBuf,
    update: bool,
) -> Vec<Result<ImportDatResult>> {
    let mut results = vec![];

    let is_zip = file_path
        .extension()
        .map(|ext| ext.to_str().unwrap().to_lowercase() == ZIP_EXTENSION)
        .unwrap_or(false);

    if is_zip {
        let tmp_dir = match tempfile::TempDir::new() {
            Ok(d) => d,
            Err(e) => {
                results.push(Err(anyhow!(e).context("Failed to create temp directory")));
                return results;
            }
        };
        let mut reader = match get_reader_sync(file_path) {
            Ok(r) => r,
            Err(e) => {
                results.push(Err(e));
                return results;
            }
        };
        let mut zip_archive = match ZipArchive::new(&mut reader) {
            Ok(a) => a,
            Err(e) => {
                results.push(Err(anyhow!(e).context("Failed to read ZIP")));
                return results;
            }
        };
        if let Err(e) = zip_archive.extract(tmp_dir.path()) {
            results.push(Err(anyhow!(e).context("Failed to extract ZIP")));
            return results;
        }
        let mut dat_paths: Vec<PathBuf> = WalkDir::new(tmp_dir.path())
            .into_iter()
            .filter_map(|e| e.ok())
            .filter(|e| {
                e.path().is_file()
                    && e.path()
                        .extension()
                        .is_some_and(|ext| ext.to_str().unwrap().to_lowercase() == DAT_EXTENSION)
            })
            .map(|e| e.path().to_path_buf())
            .collect();
        dat_paths.sort();
        for dat_path in dat_paths {
            results.push(process_single_dat(connection, progress_bar, &dat_path, update).await);
        }
    } else {
        results.push(process_single_dat(connection, progress_bar, file_path, update).await);
    }
    results.retain(|r| !matches!(r, Ok(ImportDatResult::Skipped)));

    results
}

#[cfg(feature = "server")]
async fn process_single_dat(
    connection: &mut SqliteConnection,
    progress_bar: &ProgressBar,
    dat_path: &PathBuf,
    update: bool,
) -> Result<ImportDatResult> {
    let (datfile_xml, detector_xml) = parse_dat(progress_bar, dat_path, false).await?;
    if update
        && find_system_by_name(connection, &datfile_xml.system.name)
            .await
            .is_none()
    {
        return Ok(ImportDatResult::Skipped);
    }
    import_dat(
        connection,
        progress_bar,
        &datfile_xml,
        &detector_xml,
        None,
        None,
        false,
    )
    .await
    .map(|opt| match opt {
        Some(summary) => ImportDatResult::Imported(summary),
        None => ImportDatResult::UpToDate(datfile_xml.system.name.clone()),
    })
}

#[cfg(test)]
mod test_dat;
#[cfg(test)]
mod test_dat_as_is;
#[cfg(test)]
mod test_dat_custom_name;
#[cfg(test)]
mod test_dat_custom_name_revert;
#[cfg(test)]
mod test_dat_headered;
#[cfg(test)]
mod test_dat_headered_duplicate_clrmamepro;
#[cfg(test)]
mod test_dat_headered_embedded;
#[cfg(test)]
mod test_dat_headered_skipped_header;
#[cfg(test)]
mod test_dat_mame;
#[cfg(test)]
mod test_dat_mame_chd;
#[cfg(test)]
mod test_dat_mame_mixed;
#[cfg(test)]
mod test_dat_outdated_forced;
#[cfg(test)]
mod test_dat_outdated_should_do_nothing;
#[cfg(test)]
mod test_dat_parent_clone;
#[cfg(test)]
mod test_dat_updated;
#[cfg(test)]
mod test_dat_updated_orphan_archive;
#[cfg(test)]
mod test_dat_updated_orphan_archive_mismatch;
#[cfg(test)]
mod test_dat_updated_orphan_chd;
#[cfg(test)]
mod test_dat_updated_orphan_chd_mismatch;
#[cfg(test)]
mod test_regions_france_germany;
#[cfg(test)]
mod test_regions_world;
#[cfg(test)]
mod test_zip_nested;
#[cfg(test)]
mod test_zip_update_known_system;
#[cfg(test)]
mod test_zip_update_unknown_system;