dupehell 0.5.1

Synthetic multi-domain dataset generator for record linkage benchmarking — Educational Use Only
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

"""Check hell difficulty exact_dup pair availability across domains."""
import polars as pl
from itertools import combinations
import sys, io
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')

from dupehell import generate

DOMAINS = ["kyc", "healthcare", "fintech", "publishing", "ecommerce"]
OUT = "hn_quality_output"

for domain in DOMAINS:
    result = generate(domain, size=2000, seed=42, difficulty="hell",
                      output_dir=OUT, output_format="ipc")
    df = pl.read_ipc(result.dataset)
    gt = pl.read_ipc(result.ground_truth)

    print(f"\n=== {domain} (hell, size=2000) ===")
    mc = gt.group_by("match_type").agg(pl.len()).to_pandas().set_index("match_type")["len"].to_dict()
    
    for mt, cnt in sorted(mc.items()):
        if mt == "canary":
            continue
        rids = set(gt.filter(pl.col("match_type") == mt)["record_id"].to_list())
        sub = df.filter(pl.col("record_id").is_in(rids))
        g = {}
        for r in sub.iter_rows(named=True):
            g.setdefault(r["master_id"], []).append(r["record_id"])
        multi = {k: v for k, v in g.items() if len(v) >= 2}
        pairs = sum(len(list(combinations(v, 2))) for v in multi.values())
        print(f"  {mt:>12}: records={len(rids):>4}, masters={len(g):>4}, multi={len(multi):>3}, pairs={pairs:>5}")