neser 0.3.1

""" Scraper for nescartdb.com """
import re
from pathlib import Path
from typing import Dict, Iterator, List, Optional, Union
from urllib.request import Request, urlopen

from bs4 import BeautifulSoup

try:
    from .rom_database import HardwareType, ControllerType, RomDbKey, hardware_from_console_type_and_region
except ImportError:  # pragma: no cover - allow running as a script
    from rom_database import HardwareType, ControllerType, RomDbKey, hardware_from_console_type_and_region

BASE_URL = "https://nescartdb.com/profile/view/{}"


class NesCartDb:
    """Iterator-like scraper for nescartdb.com profiles.

    Initialize with a list of integer IDs (or a single ID). Call
    `next_record()` repeatedly to fetch and parse each profile; returns a
    dict of string values (keys only present when available). Returns None
    when exhausted.
    """

    def __init__(self, ids: Union[str, List[int], int], base_url: Optional[str] = None):
        """Create a scraper backed by an iterable of numeric ids.

        ids may be:
        - A string 'all' to iterate 0..4800
        - A string range '123-456' to iterate that inclusive range
        - A string containing a single number '123'
        - An int or a list of ints (kept for backward compatibility)
        """
        # Normalize input into an iterator of ints
        max_id = 4800
        ids_list: List[int]
        if isinstance(ids, str):
            s = ids.strip().lower()
            if s == "all":
                ids_list = list(range(1, max_id + 1))
            else:
                tokens = [token.strip() for token in s.split(",") if token.strip()]
                ids_list = []
                for token in tokens:
                    if token == "all":
                        raise ValueError("'all' cannot be combined with other ids")
                    m = re.match(r"^(\d+)-(\d+)$", token)
                    if m:
                        start, end = int(m.group(1)), int(m.group(2))
                        if start > end:
                            start, end = end, start
                        start = max(0, start)
                        end = min(max_id, end)
                        ids_list.extend(range(start, end + 1))
                    elif token.isdigit():
                        ids_list.append(int(token))
                    else:
                        raise ValueError(f"invalid ids string: {ids}")
        elif isinstance(ids, int):
            ids_list = [ids]
        else:
            # Assume it's already an iterable/list of ints
            ids_list = list(ids)

        self._remaining = len(ids_list)
        self._ids_iter: Iterator[int] = iter(ids_list)
        self._base_url = base_url or BASE_URL
        self._cache_dir = Path(".html_cache")

    def _fetch_html(self, url: str) -> str:
        req = Request(url, headers={"User-Agent": "neser-rom-scraper/1.0"})
        with urlopen(req, timeout=30) as resp:
            return resp.read().decode("utf-8", errors="replace")

    def _fetch_html_with_cache(self, rom_id: int, url: str) -> str:
        cache_path = self._cache_dir / f"{rom_id}.html"
        if cache_path.exists():
            return cache_path.read_text(encoding="utf-8", errors="replace")

        html = self._fetch_html(url)
        self._cache_dir.mkdir(parents=True, exist_ok=True)
        cache_path.write_text(html, encoding="utf-8")
        return html

    @staticmethod
    def _normalize_label(text: Optional[str]) -> str:
        if not text:
            return ""
        return re.sub(r"\s+", " ", text.strip().rstrip(":")).lower()

    @staticmethod
    def _extract_key_value_pairs(soup: BeautifulSoup) -> Dict[str, str]:
        values: Dict[str, str] = {}
        for table in soup.find_all("table"):
            for row in table.find_all("tr"):
                cells = row.find_all(["th", "td"])
                if len(cells) != 2:
                    continue
                label = NesCartDb._normalize_label(cells[0].get_text(" ", strip=True))
                value = cells[1].get_text(" ", strip=True)
                if not label or not value:
                    continue
                values.setdefault(label, value)
        return values

    @staticmethod
    def _table_title(table) -> Optional[str]:
        first_row = table.find("tr")
        if not first_row:
            return None
        cells = first_row.find_all(["th", "td"])
        if len(cells) != 1:
            return None
        return NesCartDb._normalize_label(cells[0].get_text(" ", strip=True))

    @staticmethod
    def _find_titled_table(soup: "BeautifulSoup", title: str):
        normalized = NesCartDb._normalize_label(title)
        for table in soup.find_all("table"):
            if NesCartDb._table_title(table) == normalized:
                return table
        return None

    @staticmethod
    def _parse_eeprom_size_from_chip_info(table) -> Optional[int]:
        if table is None:
            return None
        for row in table.find_all("tr"):
            cells = [cell.get_text(" ", strip=True) for cell in row.find_all(["th", "td"]) ]
            if len(cells) < 4:
                continue
            designation = cells[0].lower()
            chip_type = cells[3].lower()
            if "eeprom" not in designation and "eeprom" not in chip_type:
                continue
            m = re.search(r"(\d+)\s*(kb|mb)", chip_type)
            if not m:
                return None
            size = int(m.group(1))
            unit = m.group(2)
            if unit == "kb":
                return (size * 1024) // 8
            if unit == "mb":
                return (size * 1024 * 1024) // 8
        return None

    @staticmethod
    def _parse_rom_details(table) -> Dict[str, str]:
        results: Dict[str, str] = {}
        if table is None:
            return results
        for row in table.find_all("tr"):
            cells = [cell.get_text(" ", strip=True) for cell in row.find_all(["th", "td"]) ]
            if len(cells) < 4:
                continue
            kind = NesCartDb._normalize_label(cells[0])
            size = cells[2].strip()
            crc = cells[3].strip()
            if kind.startswith("prg") or kind.startswith("chr"):
                results["crc"] = crc
            if kind == "roms combined":
                results["crc"] = size
            elif kind.startswith("prg"):
                old = results.get(RomDbKey.PRG_ROM_SIZE) or 0
                results[RomDbKey.PRG_ROM_SIZE] = old + NesCartDb._parse_size(size)
            elif kind.startswith("chr"):
                old = results.get(RomDbKey.CHR_ROM_SIZE) or 0
                results[RomDbKey.CHR_ROM_SIZE] = old + NesCartDb._parse_size(size)
        return results

    @staticmethod
    def _first_value(values: Dict[str, str], labels) -> Optional[str]:
        for label in labels:
            key = NesCartDb._normalize_label(label)
            if key in values:
                return values[key]
        return None

    @staticmethod
    def _parse_video_system(region_value: Optional[str]) -> Optional[str]:
        if not region_value:
            return None
        m = re.search(r"\b(NTSC|PAL)\b", region_value, re.IGNORECASE)
        if m:
            return m.group(1).upper()
        return None

    @staticmethod
    def _parse_int(value: Optional[str]) -> Optional[int]:
        if not value:
            return None
        m = re.search(r"(\d+)", value.replace(",", ""))
        if not m:
            return None
        return int(m.group(1))

    @staticmethod
    def _parse_size(value: Optional[str]) -> Optional[int]:
        if not value:
            return None
        cleaned = value.replace(",", "").strip().lower()
        m = re.search(r"(\d+)(?:\s*(kb|mb))?", cleaned)
        if not m:
            return None
        size = int(m.group(1))
        unit = m.group(2)
        if unit == "kb":
            return size * 1024
        if unit == "mb":
            return size * 1024 * 1024
        return size

    @staticmethod
    def _parse_yes_no(value: Optional[str]) -> Optional[int]:
        if not value:
            return None
        normalized = value.strip().lower()
        if normalized in {"yes", "y", "true"}:
            return 1
        if normalized in {"no", "n", "false"}:
            return 0
        return None

    @staticmethod
    def _match_non_standard_controller(rom_id: int, value: str,) -> Optional[int]:
        normalized = value.strip().lower()
        if "4-player adapter" in normalized:
            # Smash T.V. is double fisted, which implies a four score
            if rom_id in [927, 1306, 2025]:
                return ControllerType.DOUBLE_FISTED.value
            # Famicom variants of four score
            elif rom_id in [2163, 2236, 3601]:
                return ControllerType.FAMICOM_FOUR_PLAYERS_SIMPLE.value
            return ControllerType.NES_FOUR_SCORE.value
        elif "zapper light gun" in normalized:
            return ControllerType.ZAPPER_4017.value
        # 525, 600, 780, 910, 928 should be B not A
        elif "power pad" in normalized or "family fun fitness mat" in normalized:
            # Only Athletic world (525) and Street Cop (928) used B side
            if rom_id in [525, 928]:
                return ControllerType.POWER_PAD_SIDE_B.value
            return ControllerType.POWER_PAD_SIDE_A.value
        # All Japaneese titles used the A side of the mat
        elif "family trainer mat" in normalized:
            # Special case with the Pokkun Moguraa whack-a-mole mat and mole mallet
            if rom_id == 4686:
                return ControllerType.POKKUN_MOGURAA_TAP_MAT.value
            return ControllerType.FAMILY_TRAINER_SIDE_A.value
        # ROB is one and the same hardware but need to patch as they differ in nes20db
        elif "r. o. b." in normalized:
            # Gyromite ROB
            if rom_id in [266, 584, 785, 1286, 2789, 2788, 2354, 4063, 4423, 4524, 4542, 4543]:
                return ControllerType.ROB_GYROMITE.value
            # Stack Up ROB
            else:
                return ControllerType.ROB_STACK_UP.value
        elif "3-d glasses" in normalized:
            return ControllerType.FAMICOM_3D_SYSTEM.value
        elif "power glove" in normalized:
            return ControllerType.POWER_GLOVE.value
        elif "vaus controller" in normalized:
            # Famicom version
            if rom_id == 1757:
                return ControllerType.ARKANOID_VAUS_FAMICOM.value
            return ControllerType.ARKANOID_VAUS_NES.value
        elif "miracle piano" in normalized:
            return ControllerType.MIRACLE_PIANO.value
        elif "aladdin deck enhancer" in normalized:
            return ControllerType.ALADDIN_DECK_ENHANCER.value
        elif "barcode battler" in normalized:
            return ControllerType.SUNSOFT_BARCODE_BATTLER.value
        elif "top rider bike" in normalized:
            return ControllerType.TOP_RIDER.value
        elif "konami hypershot" in normalized:
            return ControllerType.KONAMI_HYPER_SHOT.value
        elif "mahjong controller" in normalized:
            return ControllerType.JISSEN_MAHJONG.value
        elif "battle box" in normalized:
            return ControllerType.IGS_STORAGE_BATTLE_BOX.value
        elif "racermate bike" in normalized:
            return ControllerType.RACERMATE_BICYCLE.value
        elif "family keyboard" in normalized:
            return ControllerType.FAMILY_BASIC_KEYBOARD_RECORDER.value
        elif "party tap" in normalized:
            return ControllerType.YONEZAWA_PARTY_TAP.value
        elif "oeka kids tablet" in normalized:
            return ControllerType.OEKA_KIDS_TABLET.value
        elif "pachinko controller" in normalized:
            return ControllerType.COCONUTS_PACHINKO.value
        elif "u-force" in normalized:
            return ControllerType.U_FORCE.value
        return None

    @staticmethod
    def _parse_periphereals(rom_id: int, value: str) -> Optional[int]:
        # If there are more than one value, one is bound to be Famicom/NES controller,
        # so ignore that. Accommodate for an empty value bug on the web page
        value = ",".join(v.strip() for v in value.split(",") if v.strip())
        if ',' in value:
            for val in value.split(','):
                matched = NesCartDb._match_non_standard_controller(rom_id, val)
                if matched is not None:
                    return matched
            print(f"\nUnrecognized peripherals value: '{value}'")
            exit(1)
        matched = NesCartDb._match_non_standard_controller(rom_id, value)
        if matched is not None:
            return matched
        if "nes controller" in value.lower() or "famicom controller" in value.lower():
            return ControllerType.STANDARD_CONTROLLERS
        print(f"\nUnrecognized peripherals value: '{value}'")
        exit(1)

    def _build_result(self, rom_id: int, html: str) -> Optional[Dict[str, str]]:
        soup = BeautifulSoup(html, "html.parser")
        invalid_header = soup.find("h3")
        if invalid_header and invalid_header.get_text(strip=True) == "Invalid profile specified!":
            return None
        kv = self._extract_key_value_pairs(soup)
        rom_details = self._parse_rom_details(self._find_titled_table(soup, "ROM Details"))
        chip_info = self._find_titled_table(soup, "Detailed Chip Info")

        game_name = None
        title_tag = soup.find("title")
        if title_tag and title_tag.text:
            game_name = title_tag.text.split(" - ")[0].strip()
        h1_tag = soup.find("h1")
        if h1_tag and h1_tag.text:
            game_name = h1_tag.text.strip()

        result: Dict[str, str] = {}
        if game_name:
            result[RomDbKey.NAME.value] = game_name
        if rom_details.get("crc"):
            result[RomDbKey.CRC.value] = rom_details.get("crc")
        region_text = self._first_value(kv, ["Region"])
        video_system = self._parse_video_system(region_text)
        if video_system == "PAL":
            result[RomDbKey.HARDWARE.value] = HardwareType.NES_PAL.value
        else:
            hw = hardware_from_console_type_and_region("0", "0", country=region_text)
            result[RomDbKey.HARDWARE.value] = hw if hw is not None else HardwareType.NES_NTSC.value
        mapper = self._first_value(kv, ["iNES Mapper", "Mapper"])
        submapper = self._first_value(kv, ["Submapper", "SubMapper"])
        chr_ram = self._first_value(kv, ["CHR RAM", "CHR-RAM", "VRAM"])
        work_ram = self._first_value(kv, ["WRAM", "Work RAM"])
        eeprom_size = self._parse_eeprom_size_from_chip_info(chip_info)
        batt = self._first_value(kv, ["Battery present", "Battery"])
        # In iNES 2.0, battery means "battery or other non-volatile memory"
        if eeprom_size is not None and eeprom_size > 0:
            batt = '1'
        peri = self._first_value(kv, ["Peripherals", "Controllers"])
        if mapper:
            result[RomDbKey.MAPPER.value] = self._parse_int(mapper)
        if submapper:
            result[RomDbKey.SUBMAPPER.value] = self._parse_int(submapper)
        if rom_details.get(RomDbKey.PRG_ROM_SIZE):
            result[RomDbKey.PRG_ROM_SIZE.value] = rom_details.get(RomDbKey.PRG_ROM_SIZE)
        if rom_details.get(RomDbKey.CHR_ROM_SIZE):
            result[RomDbKey.CHR_ROM_SIZE.value] = rom_details.get(RomDbKey.CHR_ROM_SIZE)
        if chr_ram:
            result[RomDbKey.CHR_RAM_SIZE.value] = self._parse_size(chr_ram)
        if work_ram:
            if self._parse_yes_no(batt):
                result[RomDbKey.PRG_NVRAM_SIZE.value] = self._parse_size(work_ram)
            else:
                result[RomDbKey.PRG_RAM_SIZE.value] = self._parse_size(work_ram)
                if eeprom_size is not None:
                    result[RomDbKey.PRG_NVRAM_SIZE.value] = eeprom_size
        if batt:
            result[RomDbKey.BATTERY.value] = self._parse_yes_no(batt)
        if peri:
            peri_value = self._parse_periphereals(rom_id, peri)
            if peri_value:
                result[RomDbKey.EXPANSION_TYPE.value] = peri_value
        # Peripherals/Controller fields are text and have no matching DB column.

        return result

    def num_left(self) -> int:
        """Return the number of records left to parse."""
        return self._remaining

    def _patch(self, rom_id: int, record: Optional[Dict[str, str]]) -> None:
        """Apply hardcoded patches for known bad/missing data."""
        # Startropics I and II (41, 814, 1896, 2449, 2769, 2780, 4171, 4365) has a 1kB PRG RAM in
        # the MMC6 chip.
        # As it also has a battery, it will be converted to NVRAM later in the parsing
        if rom_id in [41, 814, 1896, 2449, 2769, 2780, 4171, 4365]:
            record[RomDbKey.PRG_RAM_SIZE.value] = 1024
        # Pyramid with ROM_IDs 219 and 315: nes20db and nescart say different mappers.
        # 0 vs 79. However, these should be interchangable without any banking (8kB CHR)
        if rom_id in [219, 315]:
            record[RomDbKey.MAPPER.value] = 0
        # Gauntlet (ROM_ID 473), came in both Mapper 4 and 206 variants (same CRC)
        # nes20db says no 4 so let's go with that
        if rom_id in [473, 1316]:
            record[RomDbKey.MAPPER.value] = 4
        # Kyonshiizu 2 (ROM_ID 1559, 3151, 3152, 3153, 3300) has a X1-005 with internal Save RAM
        # Taito Grand Prix: Eikou e no License (ROM_ID 1758) too
        # Fudou Myouou Den (ROM_ID 1762) too
        # Mirai Shinwa Jarvas (ROM_ID 1763, 3163) too
        # Kyuukyoku Harikiri Stadium (ROM_IDs 1765, 1766, 3071, 3147, 3148, 3149, 3150, 3303) too
        # Yamamura Misa Suspense: Kyouto Ryuu no Tera Satsujin Jiken (3954) too
        # Minelvaton Saga: Ragon no Fukkatsu (3955) too
        if rom_id in [1559, 3151, 3152, 3153, 3300, 1758, 1762, 1763, 3163, 1765, 1766, 3071,
                        3147, 3148, 3149, 3150, 3303, 3954, 3955]:
            record[RomDbKey.PRG_NVRAM_SIZE.value] = 128
        # Kyuukyoku Harikiri Stadium: Heisei Gannen Ban (1767, 2254, 3161) has a Taito X1-017 with
        # Save RAM
        # Kyuukyoku Harikiri Koushien (3956) too
        # Kyuukyoku Harikiri Stadium III (3961) too
        # SD Keiji: Blader (3989) too
        if rom_id in [1767, 2254, 3161, 3956, 3961, 3989]:
            record[RomDbKey.PRG_NVRAM_SIZE.value] = 1024
        # Famicom Jump II: Saikyou no 7 Nin (ROM_ID 1734) is mapper 153 (not 16) as PRG-ROM
        # is larger than 128 kB
        if rom_id in [1734, 3280]:
            record[RomDbKey.MAPPER.value] = 153
        # # Dragon Ball Z: Kyoushuu! Saiyajin (2248) has a 128 byte EEPROM
        # # Magical Taruruuto-kun: Fantastic World!! (Version 2.0) (ROM_ID 1747, 2244) too
        # # SD Gundam Gaiden: Knight Gundam Monogatari (Version 2.0) (1746, 2249, 3079, 3080, 3081,
        # # 3082, 3083) too
        # if rom_id in [1747, 2244, 2248, 1746, 2249, 3079, 3080, 3081, 3082, 3083]:
        #     record[RomDbKey.PRG_NVRAM_SIZE.value] = 128
        # Bubble Bath Babes (ROM_ID 1838) is mapper 148 as this is an unlicensed AVE&NINA board
        if rom_id == 1838:
            record[RomDbKey.MAPPER.value] = 148
        # Skate Boy (2639) has no CHR ROM switching, so it should be mapper 0, not 4
        # El Monstruo de los Globos (2640) too
        # Booky Man (2647) too
        if rom_id in [2639, 2640, 2647]:
            record[RomDbKey.MAPPER.value] = 0
        # Galactic Crusader (2949) could be either mapper 79 or 146. Both will work.
        # Choosing 79 as 146 for a non-switching 8kB ROMs seems odd
        if rom_id == 2949:
            record[RomDbKey.MAPPER.value] = 79
        # Solstics have different names in different languages, but with same CRC.
        # Align on English.
        if rom_id in [1617, 1998, 2018]:
            record[RomDbKey.NAME.value] = "Solstice: The Quest for the Staff of Demnos"
        # The same for Solar Jetman
        if rom_id in [2153, 2653]:
            record[RomDbKey.NAME.value] = "Solar Jetman: Hunt for the Golden Warpship"
        # The same for Die Schlümpfe (The Smurfs)
        if rom_id in [1615, 1933, 2121]:
            record[RomDbKey.NAME.value] = "The Smurfs"
        # The same for Arch Rivals: A Basketbrawl!
        if rom_id in [1301, 2403, 4435, 4492]:
            record[RomDbKey.NAME.value] = "Arch Rivals: A Basketbrawl!"
        # And The Lion King
        if rom_id in [2037, 2704, 2886, 4500]:
            record[RomDbKey.NAME.value] = "Disney's The Lion King"
        # and The Jungle Book
        if rom_id in [1931, 4465]:
            record[RomDbKey.NAME.value] = "Disney's The Jungle Book"
        # and War in the Gulf
        if rom_id == 4247:
            record[RomDbKey.NAME.value] = "War In The Gulf"
        # and The Hunt for Red October
        if rom_id in [4416, 4444, 4445]:
            record[RomDbKey.NAME.value] = "The Hunt for Red October"
        # Stack-up and Block Set are the same game, but with different names in different regions.
        # Align on Stack-Up as it's more common
        if RomDbKey.NAME.value in record and record[RomDbKey.NAME.value] == "Block Set":
            record[RomDbKey.NAME.value] = "Stack-Up"
        # Bakushou!! Jinsei Gekijou 2 (1764) has a X1-005 chip which means mapper 33
        if rom_id == 1764:
            record[RomDbKey.MAPPER.value] = 33
        # The Money Game (3458) Uses MMC1A, which is mapper 155, not 1
        # Tatakae!! Ramen Man: Sakuretsu Choujin 102 Gei (3736) too
        if rom_id in [3458, 3736]:
            record[RomDbKey.MAPPER.value] = 155
        # Advanced Dungeons & Dragons Heroes of the Lance Prototype (4625) has a battery (PCB image)
        # Taro's Quest Prototype (4639) too
        # Thomas The Tank Engine & Friends Prototype (4640) too
        if rom_id in [4625, 4639, 4640]:
            record[RomDbKey.BATTERY.value] = 1
        # Days of Thunder and its Protoype have the same CRC. Align on non-prototype.
        if rom_id in [4729]:
            record[RomDbKey.BATTERY.value] = 0
            record[RomDbKey.PRG_NVRAM_SIZE.value] = 0
        # Same for Where in Time is Carmen Sandiego?
        if rom_id in [4642]:
            record[RomDbKey.BATTERY.value] = 0
        # Japaneese version of Gyromite is called Gyro, but has the same CRC. Align on Gyromite.
        if rom_id == 4063:
            record[RomDbKey.NAME.value] = "Gyromite"
        # One of The Simpsons:  Bart vs. The Space Mutants is spelled with a small t in 'the'
        if rom_id == 4394:
            record[RomDbKey.NAME.value] = "The Simpsons:  Bart vs. The Space Mutants"
        # Same game different names in different regions. Align on the more common name,
        # which is the English one.
        if rom_id in [4424, 4461]:
            record[RomDbKey.NAME.value] = "Goal! 2"

    def next_record(self) -> Optional[Dict[str, str]]:
        """Fetch and return the next parsed profile record, or None if done."""
        record  = None
        while record is None:
            try:
                rom_id = next(self._ids_iter)
            except StopIteration:
                return None

            url = self._base_url.format(rom_id)
            html = self._fetch_html_with_cache(rom_id, url)
            record = self._build_result(rom_id, html)
            if record is not None:
                self._patch(rom_id, record)
            self._remaining -= 1
        return record