kontochronik 0.2.0

//! Import adapter for the CSV export of GLS Bank online banking.
//!
//! This module only knows how to read what the bank currently exports.
//! The `kontochronik` archive file has its own, stable schema;
//! If GLS changes its export format, only this adapter has to follow.

use anyhow::bail;
use csv::ReaderBuilder;
use rust_decimal::Decimal;
use serde::Deserialize;
use time::Date;

use crate::{archive::ArchiveRecord, import::Importer, util::canonical_text};

pub const CSV_DELIMITER: u8 = b';';

/// One row of the GLS CSV export, verbatim.
#[derive(Debug, Deserialize)]
pub struct GlsRecord {
    #[serde(rename = "Bezeichnung Auftragskonto")]
    pub account_description: String,

    #[serde(rename = "IBAN Auftragskonto")]
    pub account_iban: String,

    #[serde(rename = "BIC Auftragskonto")]
    pub account_bic: String,

    #[serde(rename = "Bankname Auftragskonto")]
    pub account_bank_name: String,

    #[serde(rename = "Buchungstag", with = "german_date")]
    pub booking_date: Date,

    #[serde(rename = "Valutadatum", with = "german_date")]
    pub value_date: Date,

    #[serde(rename = "Name Zahlungsbeteiligter")]
    pub participant_name: Option<String>,

    #[serde(rename = "IBAN Zahlungsbeteiligter")]
    pub participant_iban: Option<String>,

    #[serde(rename = "BIC (SWIFT-Code) Zahlungsbeteiligter")]
    pub participant_bic: Option<String>,

    #[serde(rename = "Buchungstext")]
    pub booking_text: String,

    #[serde(rename = "Verwendungszweck")]
    pub purpose: Option<String>,

    #[serde(rename = "Betrag", with = "german_number")]
    pub amount: Decimal,

    #[serde(rename = "Waehrung")]
    pub currency: String,

    #[serde(rename = "Saldo nach Buchung", with = "german_number")]
    pub balance_after_booking: Decimal,

    #[serde(rename = "Glaeubiger ID")]
    pub creditor_id: Option<String>,

    #[serde(rename = "Mandatsreferenz")]
    pub mandate_reference: Option<String>,

    #[serde(rename = "Bemerkung", default)]
    pub note: Option<String>,

    #[serde(rename = "Gekennzeichneter Umsatz", default)]
    pub flagged: Option<String>,
}

impl TryFrom<GlsRecord> for ArchiveRecord {
    type Error = anyhow::Error;

    fn try_from(gls: GlsRecord) -> Result<Self, Self::Error> {
        let GlsRecord {
            account_description,
            account_iban,
            account_bic,
            account_bank_name,
            booking_date,
            value_date,
            participant_name,
            participant_iban,
            participant_bic,
            booking_text,
            purpose,
            amount,
            currency,
            balance_after_booking,
            creditor_id,
            mandate_reference,
            note,
            flagged,
        } = gls;
        // Portal annotations are out of scope for the archive;
        if let Some(note) = clean_optional(note) {
            bail!("\"Bemerkung\" contains data ({note:?}) — portal annotations are not archived");
        }
        if let Some(flagged) = clean_optional(flagged) {
            bail!(
                "\"Gekennzeichneter Umsatz\" contains data ({flagged:?}) — portal annotations are not archived"
            );
        }
        Ok(Self {
            booking_date,
            value_date,
            amount,
            currency: canonical_text(&currency),
            balance_after_booking,
            participant_name: clean_optional(participant_name),
            participant_iban: clean_optional(participant_iban),
            participant_bic: clean_optional(participant_bic),
            booking_text: canonical_text(&booking_text),
            purpose: clean_optional(purpose),
            creditor_id: clean_optional(creditor_id),
            mandate_reference: clean_optional(mandate_reference),
            account_description: canonical_text(&account_description),
            account_iban: canonical_text(&account_iban),
            account_bic: canonical_text(&account_bic),
            account_bank_name: canonical_text(&account_bank_name),
            fingerprint: None,
        })
    }
}

fn clean_optional(value: Option<String>) -> Option<String> {
    value.map(|s| canonical_text(&s)).filter(|s| !s.is_empty())
}

/// Every column the GLS export is known to contain.
/// Serves as a tripwire:
/// an unknown column means the bank changed its format, and silently
/// ignoring it could silently ignore data.
const KNOWN_HEADERS: [&str; 18] = [
    "Bezeichnung Auftragskonto",
    "IBAN Auftragskonto",
    "BIC Auftragskonto",
    "Bankname Auftragskonto",
    "Buchungstag",
    "Valutadatum",
    "Name Zahlungsbeteiligter",
    "IBAN Zahlungsbeteiligter",
    "BIC (SWIFT-Code) Zahlungsbeteiligter",
    "Buchungstext",
    "Verwendungszweck",
    "Betrag",
    "Waehrung",
    "Saldo nach Buchung",
    "Glaeubiger ID",
    "Mandatsreferenz",
    "Bemerkung",
    "Gekennzeichneter Umsatz",
];

/// The GLS import adapter:
/// plugs the GLS export wire format into [`crate::import`].
pub struct GlsImporter;

impl Importer for GlsImporter {
    fn parse(&self, bytes: &[u8]) -> anyhow::Result<Vec<ArchiveRecord>> {
        parse_gls_csv(bytes)?
            .into_iter()
            .map(ArchiveRecord::try_from)
            .collect()
    }
}

/// Parses the raw bytes of a GLS export file.
/// Accepts UTF-8 (with or without BOM) and falls back to Windows-1252
/// (the traditional encoding of German bank exports)
/// when the content is not valid UTF-8.
/// Rejects files with columns this adapter doesn't know.
fn parse_gls_csv(bytes: &[u8]) -> anyhow::Result<Vec<GlsRecord>> {
    let text = read_to_utf8(bytes)?;
    let mut rdr = ReaderBuilder::new()
        .delimiter(CSV_DELIMITER)
        .from_reader(text.as_bytes());
    let headers = rdr.headers()?.clone();
    for header in &headers {
        let header = header.trim();
        if !KNOWN_HEADERS.contains(&header) {
            bail!(
                "GLS export contains unknown column {header:?} — \
                 the import adapter must be updated before this data can be archived"
            );
        }
    }
    let mut records = vec![];
    for result in rdr.deserialize() {
        records.push(result?);
    }
    Ok(records)
}

fn read_to_utf8(bytes: &[u8]) -> anyhow::Result<String> {
    if bytes.starts_with(&[0xFF, 0xFE]) || bytes.starts_with(&[0xFE, 0xFF]) {
        bail!("UTF-16 encoded files are not supported");
    }
    let bytes = bytes.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(bytes);
    Ok(match std::str::from_utf8(bytes) {
        Ok(s) => s.to_owned(),
        Err(_) => decode_windows_1252(bytes),
    })
}

fn decode_windows_1252(bytes: &[u8]) -> String {
    bytes.iter().map(|&byte| windows_1252_char(byte)).collect()
}

fn windows_1252_char(byte: u8) -> char {
    // Only 0x80..=0x9F differ from Latin-1; every other byte maps to U+00XX.
    // The five undefined code points keep their Latin-1 interpretation.
    match byte {
        0x80 => '€',
        0x82 => '‚',
        0x83 => 'ƒ',
        0x84 => '„',
        0x85 => '…',
        0x86 => '†',
        0x87 => '‡',
        0x88 => 'ˆ',
        0x89 => '‰',
        0x8A => 'Š',
        0x8B => '‹',
        0x8C => 'Œ',
        0x8E => 'Ž',
        0x91 => '‘',
        0x92 => '’',
        0x93 => '“',
        0x94 => '”',
        0x95 => '•',
        0x96 => '–',
        0x97 => '—',
        0x98 => '˜',
        0x99 => '™',
        0x9A => 'š',
        0x9B => '›',
        0x9C => 'œ',
        0x9E => 'ž',
        0x9F => 'Ÿ',
        _ => char::from(byte),
    }
}

mod german_date {
    use serde::{Deserialize, Deserializer};
    use time::{Date, format_description::FormatItem, macros::format_description};

    const DATE_FMT: &[FormatItem<'static>] = format_description!("[day].[month].[year]");

    pub fn deserialize<'de, D>(deserializer: D) -> Result<Date, D::Error>
    where
        D: Deserializer<'de>,
    {
        let s = String::deserialize(deserializer)?;
        Date::parse(&s, DATE_FMT).map_err(serde::de::Error::custom)
    }
}

mod german_number {
    use std::str::FromStr as _;

    use rust_decimal::Decimal;
    use serde::{Deserialize, Deserializer};

    /// Liberal on purpose: strips thousands separators and blanks,
    /// so it copes with whatever number formatting the bank uses.
    fn normalize_german_number(s: &str) -> String {
        s.trim().replace([' ', '.'], "").replace(',', ".")
    }

    pub fn deserialize<'de, D>(deserializer: D) -> Result<Decimal, D::Error>
    where
        D: Deserializer<'de>,
    {
        let raw = String::deserialize(deserializer)?;
        let norm = normalize_german_number(&raw);
        Decimal::from_str(&norm).map_err(serde::de::Error::custom)
    }
}

#[cfg(test)]
mod tests {
    use std::{
        fs,
        path::{Path, PathBuf},
    };

    use super::*;

    use crate::{ImportSummary, import, read_archive};

    const GLS_HEADER: &str = "Bezeichnung Auftragskonto;IBAN Auftragskonto;\
        BIC Auftragskonto;Bankname Auftragskonto;Buchungstag;Valutadatum;\
        Name Zahlungsbeteiligter;IBAN Zahlungsbeteiligter;\
        BIC (SWIFT-Code) Zahlungsbeteiligter;Buchungstext;Verwendungszweck;\
        Betrag;Waehrung;Saldo nach Buchung;Glaeubiger ID;Mandatsreferenz;\
        Bemerkung;Gekennzeichneter Umsatz";

    const SALARY: &str = "Girokonto;DE44500105175407324931;GENODEM1GLS;\
        GLS Gemeinschaftsbank eG;01.01.2025;01.01.2025;ACME GmbH;\
        DE02300209000106531065;CMCIDEDD;Gutschrift;Gehalt Januar;\
        1.000,00;EUR;1000,00;;;;";

    const BAKERY: &str = "Girokonto;DE44500105175407324931;GENODEM1GLS;\
        GLS Gemeinschaftsbank eG;02.01.2025;02.01.2025;Bäckerei Müller;\
        DE02120300000000202051;BYLADEM1001;Kartenzahlung;Brötchen;\
        -3,50;EUR;996,50;;;;";

    const POWER: &str = "Girokonto;DE44500105175407324931;GENODEM1GLS;\
        GLS Gemeinschaftsbank eG;03.01.2025;03.01.2025;Stadtwerke;\
        DE02500105170137075030;INGDDEFF;Lastschrift;Strom Abschlag;\
        -45,00;EUR;951,50;DE98ZZZ09999999999;M-001;;";

    const COFFEE_1: &str = "Girokonto;DE44500105175407324931;GENODEM1GLS;\
        GLS Gemeinschaftsbank eG;05.01.2025;05.01.2025;Café Blau;\
        DE02100100100006820101;PBNKDEFF;Kartenzahlung;Kaffee;\
        -2,50;EUR;997,50;;;;";

    const COFFEE_2: &str = "Girokonto;DE44500105175407324931;GENODEM1GLS;\
        GLS Gemeinschaftsbank eG;05.01.2025;05.01.2025;Café Blau;\
        DE02100100100006820101;PBNKDEFF;Kartenzahlung;Kaffee;\
        -2,50;EUR;995,00;;;;";

    fn gls_content(rows: &[&str]) -> String {
        let mut content = GLS_HEADER.to_owned();
        for row in rows {
            content.push('\n');
            content.push_str(row);
        }
        content.push('\n');
        content
    }

    fn gls_file(dir: &Path, name: &str, rows: &[&str]) -> PathBuf {
        let path = dir.join(name);
        fs::write(&path, gls_content(rows)).unwrap();
        path
    }

    #[test]
    fn utf8_passes_through_and_bom_is_stripped() {
        assert_eq!(read_to_utf8("Bäckerei".as_bytes()).unwrap(), "Bäckerei");
        assert_eq!(
            read_to_utf8(b"\xEF\xBB\xBFB\xC3\xA4ckerei").unwrap(),
            "Bäckerei"
        );
    }

    #[test]
    fn windows_1252_is_decoded() {
        assert_eq!(
            read_to_utf8(b"B\xE4ckerei M\xFCller").unwrap(),
            "Bäckerei Müller"
        );
        assert_eq!(read_to_utf8(b"5 \x80").unwrap(), "5 €");
    }

    #[test]
    fn utf16_is_rejected() {
        assert!(read_to_utf8(b"\xFF\xFEB\x00").is_err());
    }

    #[test]
    fn merges_overlapping_exports() {
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        // Rows newest first, like the online banking export.
        let first = gls_file(dir.path(), "export1.csv", &[BAKERY, SALARY]);
        let second = gls_file(dir.path(), "export2.csv", &[POWER, BAKERY]);

        let summary = import(&GlsImporter, &first, &archive_path).unwrap();
        assert_eq!(
            summary,
            ImportSummary {
                imported: 2,
                duplicates: 0,
                total: 2
            }
        );

        let summary = import(&GlsImporter, &second, &archive_path).unwrap();
        assert_eq!(
            summary,
            ImportSummary {
                imported: 1,
                duplicates: 1,
                total: 3
            }
        );

        let records = read_archive(&archive_path).unwrap();
        let dates: Vec<String> = records
            .iter()
            .map(|record| record.booking_date.to_string())
            .collect();
        assert_eq!(dates, ["2025-01-03", "2025-01-02", "2025-01-01"]);
        assert!(
            records
                .iter()
                .all(|record| record.fingerprint.as_ref().is_some_and(|fp| fp.len() == 32))
        );
        assert_eq!(
            records[1].participant_name.as_deref(),
            Some("Bäckerei Müller")
        );
        assert_eq!(records[2].amount.to_string(), "1000.00");
    }

    #[test]
    fn import_is_idempotent() {
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let export = gls_file(dir.path(), "export.csv", &[BAKERY, SALARY]);

        import(&GlsImporter, &export, &archive_path).unwrap();
        let before = fs::read(&archive_path).unwrap();

        let summary = import(&GlsImporter, &export, &archive_path).unwrap();
        assert_eq!(
            summary,
            ImportSummary {
                imported: 0,
                duplicates: 2,
                total: 2
            }
        );
        assert_eq!(fs::read(&archive_path).unwrap(), before);
    }

    #[test]
    fn identical_bookings_are_distinguished_by_occurrence() {
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        // Two card payments identical in everything the fingerprint hashes;
        // only the balance differs. The narrower export contains just one.
        let narrow = gls_file(dir.path(), "narrow.csv", &[COFFEE_1]);
        let full = gls_file(dir.path(), "full.csv", &[COFFEE_1, COFFEE_2]);

        import(&GlsImporter, &narrow, &archive_path).unwrap();
        let summary = import(&GlsImporter, &full, &archive_path).unwrap();
        assert_eq!(
            summary,
            ImportSummary {
                imported: 1,
                duplicates: 1,
                total: 2
            }
        );

        let records = read_archive(&archive_path).unwrap();
        assert_eq!(records.len(), 2);
        assert_ne!(records[0].fingerprint, records[1].fingerprint);
        assert_ne!(
            records[0].balance_after_booking,
            records[1].balance_after_booking
        );
    }

    #[test]
    fn empty_purpose_is_allowed() {
        const NO_PURPOSE: &str = "Girokonto;DE44500105175407324931;GENODEM1GLS;\
            GLS Gemeinschaftsbank eG;04.01.2025;04.01.2025;Automat;\
            DE02100100100006820101;PBNKDEFF;Bargeldauszahlung;;\
            -50,00;EUR;901,50;;;;";
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let export = gls_file(dir.path(), "export.csv", &[NO_PURPOSE]);

        let summary = import(&GlsImporter, &export, &archive_path).unwrap();
        assert_eq!(summary.imported, 1);

        let records = read_archive(&archive_path).unwrap();
        assert_eq!(records[0].purpose, None);
        assert!(
            records[0]
                .fingerprint
                .as_ref()
                .is_some_and(|fp| fp.len() == 32)
        );
    }

    #[test]
    fn rejects_filled_portal_annotations() {
        let annotated: &str = &SALARY.replace("1000,00;;;;", "1000,00;;;Notiz;");
        assert_ne!(annotated, SALARY);
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let export = gls_file(dir.path(), "export.csv", &[annotated]);

        let error = import(&GlsImporter, &export, &archive_path).unwrap_err();
        assert!(format!("{error:#}").contains("Bemerkung"));
    }

    #[test]
    fn rejects_unknown_export_columns() {
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let path = dir.path().join("export.csv");
        fs::write(&path, format!("{GLS_HEADER};Neue Spalte\n{SALARY};x\n")).unwrap();

        let error = import(&GlsImporter, &path, &archive_path).unwrap_err();
        assert!(format!("{error:#}").contains("Neue Spalte"));
    }

    #[test]
    fn accepts_exports_without_annotation_columns() {
        let legacy_header = GLS_HEADER.replace(";Bemerkung;Gekennzeichneter Umsatz", "");
        assert_ne!(legacy_header, GLS_HEADER);
        let legacy_row = SALARY.strip_suffix(";;").unwrap();
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let path = dir.path().join("export.csv");
        fs::write(&path, format!("{legacy_header}\n{legacy_row}\n")).unwrap();

        let summary = import(&GlsImporter, &path, &archive_path).unwrap();
        assert_eq!(summary.imported, 1);
    }

    #[test]
    fn rejects_mixed_accounts() {
        let other_account: &str =
            &SALARY.replace("DE44500105175407324931", "DE02120300000000202051");
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let export = gls_file(dir.path(), "export.csv", &[BAKERY, other_account]);

        assert!(import(&GlsImporter, &export, &archive_path).is_err());
        assert!(!archive_path.exists());
    }

    #[test]
    fn rejects_mixed_currencies() {
        let dollars: &str = &SALARY.replace(";EUR;", ";USD;");
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let export = gls_file(dir.path(), "export.csv", &[BAKERY, dollars]);

        assert!(import(&GlsImporter, &export, &archive_path).is_err());
        assert!(!archive_path.exists());
    }

    #[test]
    fn archives_a_foreign_currency_account() {
        let dollars: &str = &SALARY.replace(";EUR;", ";USD;");
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let export = gls_file(dir.path(), "export.csv", &[dollars]);

        let summary = import(&GlsImporter, &export, &archive_path).unwrap();
        assert_eq!(summary.imported, 1);

        let records = read_archive(&archive_path).unwrap();
        assert_eq!(records[0].currency, "USD");
    }

    #[test]
    fn rejects_amounts_with_more_than_two_decimals() {
        let sub_cent: &str = &SALARY.replace(";1.000,00;", ";1.000,001;");
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let export = gls_file(dir.path(), "export.csv", &[sub_cent]);

        let error = import(&GlsImporter, &export, &archive_path).unwrap_err();
        assert!(format!("{error:#}").contains("two decimal places"));
    }

    #[test]
    fn windows_1252_export_is_imported() {
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let path = dir.path().join("export.csv");
        let bytes: Vec<u8> = gls_content(&[BAKERY])
            .chars()
            .map(|c| u8::try_from(u32::from(c)).expect("Latin-1 test data"))
            .collect();
        fs::write(&path, bytes).unwrap();

        let summary = import(&GlsImporter, &path, &archive_path).unwrap();
        assert_eq!(summary.imported, 1);

        let records = read_archive(&archive_path).unwrap();
        assert_eq!(
            records[0].participant_name.as_deref(),
            Some("Bäckerei Müller")
        );
        assert_eq!(records[0].purpose.as_deref(), Some("Brötchen"));
    }

    #[test]
    fn verifies_stored_fingerprints() {
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let export = gls_file(dir.path(), "export.csv", &[SALARY]);
        import(&GlsImporter, &export, &archive_path).unwrap();

        // Tamper with the archived amount; the stored fingerprint no longer
        // matches the entry's content.
        let content = fs::read_to_string(&archive_path)
            .unwrap()
            .replace("01.01.2025;1000,00;ACME", "01.01.2025;999,00;ACME");
        assert!(content.contains("999,00"));
        fs::write(&archive_path, content).unwrap();

        let error = import(&GlsImporter, &export, &archive_path).unwrap_err();
        assert!(format!("{error:#}").contains("does not match"));
    }
}