dupehell 0.5.1

Synthetic multi-domain dataset generator for record linkage benchmarking — Educational Use Only
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

"""Analyze what fields hard_neg records share."""
import polars as pl
from itertools import combinations
import numpy as np
import sys, io
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')

from dupehell import generate

result = generate('kyc', size=2000, seed=42, difficulty='medium', output_dir='hn_quality_output', output_format='ipc')
df = pl.read_ipc(result.dataset)
gt = pl.read_ipc(result.ground_truth)

content_cols = [c for c in df.columns if c not in ('record_id', 'master_id', 'entity_type', 'domain')]

hn_rids = set(gt.filter(pl.col('match_type') == 'hard_neg')['record_id'].to_list())
hn_df = df.filter(pl.col('record_id').is_in(hn_rids))

rmap = {}
midmap = {}
for row in hn_df.iter_rows(named=True):
    rmap[row['record_id']] = row
    midmap[row['record_id']] = row['master_id']

by_mid = {}
for rid, mid in midmap.items():
    by_mid.setdefault(mid, []).append(rid)

mids = list(by_mid.keys())

# For each content column, compute what fraction of hard_neg pairs share the same value
same_col_counts = {c: 0 for c in content_cols}
total_pairs = 0

for i in range(len(mids)):
    for j in range(i+1, len(mids)):
        for a in by_mid[mids[i]]:
            for b in by_mid[mids[j]]:
                total_pairs += 1
                r1, r2 = rmap[a], rmap[b]
                for c in content_cols:
                    v1, v2 = r1[c], r2[c]
                    if v1 is not None and v2 is not None and v1 == v2:
                        same_col_counts[c] = same_col_counts[c] + 1

print(f"\nTotal hard_neg pairs compared: {total_pairs}")
print(f"\nColumn-level agreement in hard_neg pairs (sorted by agreement rate):\n")
sorted_cols = sorted(content_cols, key=lambda c: same_col_counts[c]/total_pairs, reverse=True)
for c in sorted_cols:
    rate = same_col_counts[c] / total_pairs if total_pairs > 0 else 0
    print(f"  {c:>40}: {same_col_counts[c]:>4}/{total_pairs} = {rate:.3f}")

# Now check what exact_dup pairs look like - are they truly exact copies?
print("\n" + "=" * 70)
print("Exact_dup pair verification:")
exact_rids = set(gt.filter(pl.col('match_type') == 'exact_dup')['record_id'].to_list())
exact_df = df.filter(pl.col('record_id').is_in(exact_rids))

by_mid_exact = {}
for row in exact_df.iter_rows(named=True):
    by_mid_exact.setdefault(row['master_id'], []).append(row)

multi = {k: v for k, v in by_mid_exact.items() if len(v) >= 2}
print(f"Masters with 2+ exact_dup records: {len(multi)}")
diffs_total = 0
pairs_checked = 0
for mid, rows in multi.items():
    for a, b in combinations(range(len(rows)), 2):
        r1, r2 = rows[a], rows[b]
        diffs = sum(1 for c in content_cols if r1[c] != r2[c])
        diffs_total += diffs
        pairs_checked += 1
        if pairs_checked >= 50:
            break
    if pairs_checked >= 50:
        break
print(f"Exact_dup pairs checked: {pairs_checked}, total diffs: {diffs_total}, avg diffs/pair: {diffs_total/pairs_checked:.4f}")

# Check hell difficulty: no exact_dup pairs at all
print("\n" + "=" * 70)
print("Hell difficulty — no exact_dup pairs possible:")
result_h = generate('kyc', size=2000, seed=42, difficulty='hell', output_dir='hn_quality_output', output_format='ipc')
df_h = pl.read_ipc(result_h.dataset)
gt_h = pl.read_ipc(result_h.ground_truth)
exact_rids_h = set(gt_h.filter(pl.col('match_type') == 'exact_dup')['record_id'].to_list())
exact_df_h = df_h.filter(pl.col('record_id').is_in(exact_rids_h))
by_mid_h = {}
for row in exact_df_h.iter_rows(named=True):
    by_mid_h.setdefault(row['master_id'], []).append(row)
multi_h = {k: v for k, v in by_mid_h.items() if len(v) >= 2}
print(f"Exact_dup records: {len(exact_rids_h)}, Masters: {len(by_mid_h)}, Masters with 2+: {len(multi_h)}")

# Check if ANY record shares a master_id with another record in hell
master_counts = df_h.group_by('master_id').agg(pl.len()).filter(pl.col('len') >= 2)
print(f"Any master_id shared by 2+ records (any match_type): {master_counts.height}")