dupehell 0.5.1

Synthetic multi-domain dataset generator for record linkage benchmarking — Educational Use Only
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

import json, os, tempfile, shutil
from dupehell import *
import polars as pl

for domain in ['hospitality', 'travel', 'retail', 'gaming', 'cybersecurity']:
    outdir = tempfile.mkdtemp(prefix='vrf_')
    r = generate(domain, 1000, 42, output_dir=outdir)
    df = pl.read_parquet(r.dataset)
    gt = pl.read_parquet(r.ground_truth)
    dfg = df.join(gt.select(['record_id','match_type']), on='record_id', how='left')
    
    for pc in [c for c in df.columns if 'phone' in c.lower()]:
        nn = dfg.filter(pl.col(pc).is_not_null())
        total = len(nn)
        bad = nn.with_columns(
            pl.col(pc).cast(pl.Utf8).alias('s')
        ).filter(
            pl.col('s').str.contains(r'[a-zA-Z]')
        )
        mt_dist = bad['match_type'].value_counts().to_dict() if len(bad) else {}
        print(f'{domain}.{pc}: {len(bad)}/{total} have letters - match_types: {mt_dist}')
    
    for dc in df.columns:
        if dc in ['record_id','match_type','master_id']:
            continue
        if df[dc].dtype == pl.String:
            nn = dfg.filter(pl.col(dc).is_not_null())
            bad = nn.with_columns(
                pl.col(dc).cast(pl.Utf8).alias('s')
            ).filter(
                pl.col('s').str.contains(r'[a-zA-Z]')
            )
            if len(bad) > 0:
                mt_dist = bad['match_type'].value_counts().to_dict() if len(bad) else {}
                print(f'{domain}.{dc}: {len(bad)}/{len(nn)} have letters - match_types: {mt_dist}')
    
    shutil.rmtree(outdir, ignore_errors=True)