kontochronik 0.4.0

//! Import adapter for the CSV export of GLS Bank online banking.

use anyhow::{Context as _, bail};
use csv::ReaderBuilder;
use rust_decimal::Decimal;
use serde::Deserialize;
use time::Date;

use crate::{
    archive::ArchiveRecord,
    util::{canonical_text, clean_optional, read_to_utf8},
};

const CSV_DELIMITER: u8 = b';';

/// One row of the GLS CSV export.
/// The "Bezeichnung Auftragskonto" column (a portal label, not ledger
/// data) has no archive counterpart and no field here;
/// [`KNOWN_HEADERS`] still names it.
#[derive(Debug, Deserialize)]
struct GlsRecord {
    #[serde(rename = "IBAN Auftragskonto")]
    account_iban: String,

    #[serde(rename = "BIC Auftragskonto")]
    account_bic: String,

    #[serde(rename = "Bankname Auftragskonto")]
    account_bank_name: String,

    #[serde(rename = "Buchungstag", with = "german_date")]
    booking_date: Date,

    #[serde(rename = "Valutadatum", with = "german_date")]
    value_date: Date,

    #[serde(rename = "Name Zahlungsbeteiligter")]
    participant_name: Option<String>,

    #[serde(rename = "IBAN Zahlungsbeteiligter")]
    participant_iban: Option<String>,

    #[serde(rename = "BIC (SWIFT-Code) Zahlungsbeteiligter")]
    participant_bic: Option<String>,

    #[serde(rename = "Buchungstext")]
    booking_text: String,

    #[serde(rename = "Verwendungszweck")]
    purpose: Option<String>,

    #[serde(rename = "Betrag", with = "crate::util::german_number")]
    amount: Decimal,

    #[serde(rename = "Waehrung")]
    currency: String,

    #[serde(rename = "Saldo nach Buchung", with = "crate::util::german_number")]
    balance_after_booking: Decimal,

    #[serde(rename = "Glaeubiger ID")]
    creditor_id: Option<String>,

    #[serde(rename = "Mandatsreferenz")]
    mandate_reference: Option<String>,

    #[serde(rename = "Bemerkung", default)]
    note: Option<String>,

    #[serde(rename = "Gekennzeichneter Umsatz", default)]
    flagged: Option<String>,
}

impl TryFrom<GlsRecord> for ArchiveRecord {
    type Error = anyhow::Error;

    fn try_from(gls: GlsRecord) -> Result<Self, Self::Error> {
        let GlsRecord {
            account_iban,
            account_bic,
            account_bank_name,
            booking_date,
            value_date,
            participant_name,
            participant_iban,
            participant_bic,
            booking_text,
            purpose,
            amount,
            currency,
            balance_after_booking,
            creditor_id,
            mandate_reference,
            note,
            flagged,
        } = gls;
        // Portal annotations are out of scope for the archive;
        if let Some(note) = clean_optional(note) {
            bail!("\"Bemerkung\" contains data ({note:?}) - portal annotations are not archived");
        }
        if let Some(flagged) = clean_optional(flagged) {
            bail!(
                "\"Gekennzeichneter Umsatz\" contains data ({flagged:?}) - portal annotations are not archived"
            );
        }
        Ok(Self {
            booking_date,
            value_date,
            amount,
            currency: canonical_text(&currency),
            participant_name: clean_optional(participant_name),
            purpose: clean_optional(purpose),
            booking_text: canonical_text(&booking_text),
            participant_iban: clean_optional(participant_iban),
            participant_bic: clean_optional(participant_bic),
            creditor_id: clean_optional(creditor_id),
            mandate_reference: clean_optional(mandate_reference),
            balance_after_booking: Some(balance_after_booking),
            account_iban: account_iban
                .parse()
                .with_context(|| format!("Invalid \"IBAN Auftragskonto\" {account_iban:?}"))?,
            account_bic: clean_optional(Some(account_bic)),
            account_bank_name: clean_optional(Some(account_bank_name)),
            fingerprint: None,
        })
    }
}

/// Every column the GLS export is known to contain.
/// Serves as a tripwire:
/// an unknown column means the bank changed its format, and silently
/// ignoring it could silently ignore data.
const KNOWN_HEADERS: [&str; 18] = [
    "Bezeichnung Auftragskonto",
    "IBAN Auftragskonto",
    "BIC Auftragskonto",
    "Bankname Auftragskonto",
    "Buchungstag",
    "Valutadatum",
    "Name Zahlungsbeteiligter",
    "IBAN Zahlungsbeteiligter",
    "BIC (SWIFT-Code) Zahlungsbeteiligter",
    "Buchungstext",
    "Verwendungszweck",
    "Betrag",
    "Waehrung",
    "Saldo nach Buchung",
    "Glaeubiger ID",
    "Mandatsreferenz",
    "Bemerkung",
    "Gekennzeichneter Umsatz",
];

/// Parses the CSV export of GLS Bank online banking into archive records.
pub fn parse_gls(bytes: &[u8]) -> anyhow::Result<Vec<ArchiveRecord>> {
    parse_gls_csv(bytes)?
        .into_iter()
        .map(ArchiveRecord::try_from)
        .collect()
}

/// Rejects files with columns this adapter doesn't know.
fn parse_gls_csv(bytes: &[u8]) -> anyhow::Result<Vec<GlsRecord>> {
    let text = read_to_utf8(bytes)?;
    let mut rdr = ReaderBuilder::new()
        .delimiter(CSV_DELIMITER)
        .from_reader(text.as_bytes());
    let headers = rdr.headers()?.clone();
    for header in &headers {
        let header = header.trim();
        if !KNOWN_HEADERS.contains(&header) {
            bail!(
                "GLS export contains unknown column {header:?} - \
                 the import adapter must be updated before this data can be archived"
            );
        }
    }
    let mut records = vec![];
    for result in rdr.deserialize() {
        records.push(result?);
    }
    Ok(records)
}

mod german_date {
    use serde::{Deserialize, Deserializer};
    use time::{Date, format_description::FormatItem, macros::format_description};

    const DATE_FMT: &[FormatItem<'static>] = format_description!("[day].[month].[year]");

    pub fn deserialize<'de, D>(deserializer: D) -> Result<Date, D::Error>
    where
        D: Deserializer<'de>,
    {
        let s = String::deserialize(deserializer)?;
        Date::parse(&s, DATE_FMT).map_err(serde::de::Error::custom)
    }
}

#[cfg(test)]
mod tests {
    use std::{
        fs,
        path::{Path, PathBuf},
    };

    use super::*;

    use crate::{GapPolicy, ImportSummary, import, read_archive};

    const GLS_HEADER: &str = "Bezeichnung Auftragskonto;IBAN Auftragskonto;\
        BIC Auftragskonto;Bankname Auftragskonto;Buchungstag;Valutadatum;\
        Name Zahlungsbeteiligter;IBAN Zahlungsbeteiligter;\
        BIC (SWIFT-Code) Zahlungsbeteiligter;Buchungstext;Verwendungszweck;\
        Betrag;Waehrung;Saldo nach Buchung;Glaeubiger ID;Mandatsreferenz;\
        Bemerkung;Gekennzeichneter Umsatz";

    const SALARY: &str = "Girokonto;DE44500105175407324931;GENODEM1GLS;\
        GLS Gemeinschaftsbank eG;01.01.2025;01.01.2025;ACME GmbH;\
        DE02300209000106531065;CMCIDEDD;Gutschrift;Gehalt Januar;\
        1.000,00;EUR;1000,00;;;;";

    const BAKERY: &str = "Girokonto;DE44500105175407324931;GENODEM1GLS;\
        GLS Gemeinschaftsbank eG;02.01.2025;02.01.2025;Bäckerei Müller;\
        DE02120300000000202051;BYLADEM1001;Kartenzahlung;Brötchen;\
        -3,50;EUR;996,50;;;;";

    const POWER: &str = "Girokonto;DE44500105175407324931;GENODEM1GLS;\
        GLS Gemeinschaftsbank eG;03.01.2025;03.01.2025;Stadtwerke;\
        DE02500105170137075030;INGDDEFF;Lastschrift;Strom Abschlag;\
        -45,00;EUR;951,50;DE98ZZZ09999999999;M-001;;";

    const COFFEE_1: &str = "Girokonto;DE44500105175407324931;GENODEM1GLS;\
        GLS Gemeinschaftsbank eG;05.01.2025;05.01.2025;Café Blau;\
        DE02100100100006820101;PBNKDEFF;Kartenzahlung;Kaffee;\
        -2,50;EUR;997,50;;;;";

    const COFFEE_2: &str = "Girokonto;DE44500105175407324931;GENODEM1GLS;\
        GLS Gemeinschaftsbank eG;05.01.2025;05.01.2025;Café Blau;\
        DE02100100100006820101;PBNKDEFF;Kartenzahlung;Kaffee;\
        -2,50;EUR;995,00;;;;";

    fn gls_content(rows: &[&str]) -> String {
        let mut content = GLS_HEADER.to_owned();
        for row in rows {
            content.push('\n');
            content.push_str(row);
        }
        content.push('\n');
        content
    }

    fn gls_file(dir: &Path, name: &str, rows: &[&str]) -> PathBuf {
        let path = dir.join(name);
        fs::write(&path, gls_content(rows)).unwrap();
        path
    }

    #[test]
    fn merges_overlapping_exports() {
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        // Rows newest first, like the online banking export.
        let first = gls_file(dir.path(), "export1.csv", &[BAKERY, SALARY]);
        let second = gls_file(dir.path(), "export2.csv", &[POWER, BAKERY]);

        let summary = import(parse_gls, &first, &archive_path, GapPolicy::Reject).unwrap();
        assert_eq!(
            summary,
            ImportSummary {
                imported: 2,
                duplicates: 0,
                total: 2
            }
        );

        let summary = import(parse_gls, &second, &archive_path, GapPolicy::Reject).unwrap();
        assert_eq!(
            summary,
            ImportSummary {
                imported: 1,
                duplicates: 1,
                total: 3
            }
        );

        let records = read_archive(&archive_path).unwrap();
        let dates: Vec<String> = records
            .iter()
            .map(|record| record.booking_date.to_string())
            .collect();
        assert_eq!(dates, ["2025-01-03", "2025-01-02", "2025-01-01"]);
        assert!(
            records
                .iter()
                .all(|record| record.fingerprint.as_ref().is_some_and(|fp| fp.len() == 32))
        );
        assert_eq!(
            records[1].participant_name.as_deref(),
            Some("Bäckerei Müller")
        );
        assert_eq!(records[2].amount.to_string(), "1000.00");
    }

    #[test]
    fn import_is_idempotent() {
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let export = gls_file(dir.path(), "export.csv", &[BAKERY, SALARY]);

        import(parse_gls, &export, &archive_path, GapPolicy::Reject).unwrap();
        let before = fs::read(&archive_path).unwrap();

        let summary = import(parse_gls, &export, &archive_path, GapPolicy::Reject).unwrap();
        assert_eq!(
            summary,
            ImportSummary {
                imported: 0,
                duplicates: 2,
                total: 2
            }
        );
        assert_eq!(fs::read(&archive_path).unwrap(), before);
    }

    #[test]
    fn identical_bookings_are_distinguished_by_occurrence() {
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        // Two card payments identical in everything the fingerprint hashes;
        // only the balance differs. The narrower export contains just one.
        let narrow = gls_file(dir.path(), "narrow.csv", &[COFFEE_1]);
        let full = gls_file(dir.path(), "full.csv", &[COFFEE_1, COFFEE_2]);

        import(parse_gls, &narrow, &archive_path, GapPolicy::Reject).unwrap();
        let summary = import(parse_gls, &full, &archive_path, GapPolicy::Reject).unwrap();
        assert_eq!(
            summary,
            ImportSummary {
                imported: 1,
                duplicates: 1,
                total: 2
            }
        );

        let records = read_archive(&archive_path).unwrap();
        assert_eq!(records.len(), 2);
        assert_ne!(records[0].fingerprint, records[1].fingerprint);
        assert_ne!(
            records[0].balance_after_booking,
            records[1].balance_after_booking
        );
    }

    #[test]
    fn empty_purpose_is_allowed() {
        const NO_PURPOSE: &str = "Girokonto;DE44500105175407324931;GENODEM1GLS;\
            GLS Gemeinschaftsbank eG;04.01.2025;04.01.2025;Automat;\
            DE02100100100006820101;PBNKDEFF;Bargeldauszahlung;;\
            -50,00;EUR;901,50;;;;";
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let export = gls_file(dir.path(), "export.csv", &[NO_PURPOSE]);

        let summary = import(parse_gls, &export, &archive_path, GapPolicy::Reject).unwrap();
        assert_eq!(summary.imported, 1);

        let records = read_archive(&archive_path).unwrap();
        assert_eq!(records[0].purpose, None);
        assert!(
            records[0]
                .fingerprint
                .as_ref()
                .is_some_and(|fp| fp.len() == 32)
        );
    }

    #[test]
    fn rejects_filled_portal_annotations() {
        let annotated: &str = &SALARY.replace("1000,00;;;;", "1000,00;;;Notiz;");
        assert_ne!(annotated, SALARY);
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let export = gls_file(dir.path(), "export.csv", &[annotated]);

        let error = import(parse_gls, &export, &archive_path, GapPolicy::Reject).unwrap_err();
        assert!(format!("{error:#}").contains("Bemerkung"));
    }

    #[test]
    fn rejects_unknown_export_columns() {
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let path = dir.path().join("export.csv");
        fs::write(&path, format!("{GLS_HEADER};Neue Spalte\n{SALARY};x\n")).unwrap();

        let error = import(parse_gls, &path, &archive_path, GapPolicy::Reject).unwrap_err();
        assert!(format!("{error:#}").contains("Neue Spalte"));
    }

    #[test]
    fn accepts_exports_without_annotation_columns() {
        let legacy_header = GLS_HEADER.replace(";Bemerkung;Gekennzeichneter Umsatz", "");
        assert_ne!(legacy_header, GLS_HEADER);
        let legacy_row = SALARY.strip_suffix(";;").unwrap();
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let path = dir.path().join("export.csv");
        fs::write(&path, format!("{legacy_header}\n{legacy_row}\n")).unwrap();

        let summary = import(parse_gls, &path, &archive_path, GapPolicy::Reject).unwrap();
        assert_eq!(summary.imported, 1);
    }

    #[test]
    fn rejects_mixed_accounts() {
        let other_account: &str =
            &SALARY.replace("DE44500105175407324931", "DE02120300000000202051");
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let export = gls_file(dir.path(), "export.csv", &[BAKERY, other_account]);

        assert!(import(parse_gls, &export, &archive_path, GapPolicy::Reject).is_err());
        assert!(!archive_path.exists());
    }

    #[test]
    fn rejects_mixed_currencies() {
        let dollars: &str = &SALARY.replace(";EUR;", ";USD;");
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let export = gls_file(dir.path(), "export.csv", &[BAKERY, dollars]);

        assert!(import(parse_gls, &export, &archive_path, GapPolicy::Reject).is_err());
        assert!(!archive_path.exists());
    }

    #[test]
    fn archives_a_foreign_currency_account() {
        let dollars: &str = &SALARY.replace(";EUR;", ";USD;");
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let export = gls_file(dir.path(), "export.csv", &[dollars]);

        let summary = import(parse_gls, &export, &archive_path, GapPolicy::Reject).unwrap();
        assert_eq!(summary.imported, 1);

        let records = read_archive(&archive_path).unwrap();
        assert_eq!(records[0].currency, "USD");
    }

    #[test]
    fn rejects_amounts_with_more_than_two_decimals() {
        let sub_cent: &str = &SALARY.replace(";1.000,00;", ";1.000,001;");
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let export = gls_file(dir.path(), "export.csv", &[sub_cent]);

        let error = import(parse_gls, &export, &archive_path, GapPolicy::Reject).unwrap_err();
        assert!(format!("{error:#}").contains("two decimal places"));
    }

    #[test]
    fn windows_1252_export_is_imported() {
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let path = dir.path().join("export.csv");
        let bytes: Vec<u8> = gls_content(&[BAKERY])
            .chars()
            .map(|c| u8::try_from(u32::from(c)).expect("Latin-1 test data"))
            .collect();
        fs::write(&path, bytes).unwrap();

        let summary = import(parse_gls, &path, &archive_path, GapPolicy::Reject).unwrap();
        assert_eq!(summary.imported, 1);

        let records = read_archive(&archive_path).unwrap();
        assert_eq!(
            records[0].participant_name.as_deref(),
            Some("Bäckerei Müller")
        );
        assert_eq!(records[0].purpose.as_deref(), Some("Brötchen"));
    }

    #[test]
    fn verifies_stored_fingerprints() {
        let dir = tempfile::tempdir().unwrap();
        let archive_path = dir.path().join("archive.csv");
        let export = gls_file(dir.path(), "export.csv", &[SALARY]);
        import(parse_gls, &export, &archive_path, GapPolicy::Reject).unwrap();

        // Tamper with the archived amount; the stored fingerprint no longer
        // matches the entry's content.
        let content = fs::read_to_string(&archive_path)
            .unwrap()
            .replace("01.01.2025;1000,00;EUR;ACME", "01.01.2025;999,00;EUR;ACME");
        assert!(content.contains("999,00"));
        fs::write(&archive_path, content).unwrap();

        let error = import(parse_gls, &export, &archive_path, GapPolicy::Reject).unwrap_err();
        assert!(format!("{error:#}").contains("does not match"));
    }
}