use crate::db::{DbConfig, DbPool, Document, DupeMatch};
use crate::lsh::{DiskLSH, InMemoryLSH};
use crate::minhash::jaccard_from_signatures;
use crate::sources::{DocumentSource, SourceDupeMatch};
use crate::storage::{ConnectedEdgeStats, FilteredParentStore, MatchRecord, MatchStore};
use crate::tokenizer::compute_signature;
use crate::union_find::UnionFind;
use anyhow::{Context, Result};
use indicatif::{ProgressBar, ProgressStyle};
use rayon::prelude::*;
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Instant;
use tracing::{info, warn};
use uuid::Uuid;
#[cfg(target_os = "linux")]
fn current_rss_mb() -> Option<f64> {
let status = std::fs::read_to_string("/proc/self/status").ok()?;
let line = status.lines().find(|l| l.starts_with("VmRSS:"))?;
let kb = line.split_whitespace().nth(1)?.parse::<f64>().ok()?;
Some(kb / 1024.0)
}
#[cfg(not(target_os = "linux"))]
fn current_rss_mb() -> Option<f64> {
None
}
fn log_memory(label: &str) {
if let Some(mb) = current_rss_mb() {
info!("[MEMORY] {}: {:.1} MB ({:.2} GB)", label, mb, mb / 1024.0);
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EdgeLookupMode {
Scan,
Auto,
Shadow,
}
#[derive(Debug, Clone)]
pub struct DedupeConfig {
pub threshold: f64,
pub size_diff_threshold: f64,
pub batch_size: i64,
pub num_workers: usize,
pub use_disk_lsh: bool,
pub disk_lsh_path: Option<String>,
pub seed: u64,
pub process_all: bool,
pub fresh: bool,
pub data_dir: PathBuf,
pub skip_db_write: bool,
pub disk_phase2: bool,
pub min_content_length: i32,
pub edge_lookup_mode: EdgeLookupMode,
pub max_matches_per_doc: Option<usize>,
}
impl Default for DedupeConfig {
fn default() -> Self {
Self {
threshold: 0.8,
size_diff_threshold: 0.3,
batch_size: 10000,
num_workers: num_cpus::get(),
use_disk_lsh: true, disk_lsh_path: None,
seed: 42,
process_all: false,
fresh: false,
data_dir: PathBuf::from("./data"),
skip_db_write: false,
disk_phase2: true,
min_content_length: 500, edge_lookup_mode: EdgeLookupMode::Scan,
max_matches_per_doc: None,
}
}
}
#[derive(Debug, Clone, Copy, Default)]
struct Phase3StreamStats {
edges_streamed: u64,
nodes_seen: usize,
used_adjacency_index: bool,
peak_rss_mb: Option<f64>,
elapsed_secs: f64,
}
#[derive(Debug, Clone, Copy, Default)]
struct Phase3SyncOutcome {
connected_edges: u64,
new_children: usize,
}
struct StreamedTransitivity {
assignments: Vec<DupeMatch>,
parent_ids: HashSet<Uuid>,
child_ids: HashSet<Uuid>,
}
fn stream_connected_matches<F>(
matches_store: &MatchStore,
seed_doc_ids: &[Uuid],
mode: EdgeLookupMode,
label: &str,
pass: &str,
mut visitor: F,
) -> Result<Phase3StreamStats>
where
F: FnMut(&MatchRecord) -> Result<()>,
{
const LOG_EVERY: u64 = 1_000_000;
let start = Instant::now();
let mut edges_seen = 0u64;
let mut peak_rss_mb = current_rss_mb();
let connected_stats: ConnectedEdgeStats = {
let mut wrapped_visitor = |record: &MatchRecord| -> Result<()> {
edges_seen += 1;
if let Some(mb) = current_rss_mb() {
peak_rss_mb = Some(peak_rss_mb.map_or(mb, |peak| peak.max(mb)));
}
if edges_seen.is_multiple_of(LOG_EVERY) {
match peak_rss_mb {
Some(peak) => info!(
"Phase 3 {} for {} streamed {} edges so far; peak RSS {:.1} MB",
pass, label, edges_seen, peak
),
None => info!(
"Phase 3 {} for {} streamed {} edges so far",
pass, label, edges_seen
),
}
}
visitor(record)
};
match mode {
EdgeLookupMode::Scan => {
matches_store.visit_real_edges_connected_to(seed_doc_ids, &mut wrapped_visitor)?
}
EdgeLookupMode::Auto => matches_store
.visit_real_edges_connected_to_auto(seed_doc_ids, &mut wrapped_visitor)?,
EdgeLookupMode::Shadow => {
let scan_stats = matches_store
.visit_real_edges_connected_to(seed_doc_ids, &mut wrapped_visitor)?;
if matches_store.is_adjacency_built()? {
let indexed_start = Instant::now();
let indexed_stats = matches_store
.visit_real_edges_connected_to_indexed(seed_doc_ids, |_| Ok(()))?;
if indexed_stats.edges_streamed == scan_stats.edges_streamed
&& indexed_stats.nodes_seen == scan_stats.nodes_seen
{
info!(
"Adjacency shadow streaming check passed for {} {}: {} edges, {} nodes, index {:.2}s",
label,
pass,
indexed_stats.edges_streamed,
indexed_stats.nodes_seen,
indexed_start.elapsed().as_secs_f64()
);
} else {
warn!(
"ADJACENCY_SHADOW_STREAM_MISMATCH for {} {}: scan_edges={}, indexed_edges={}, scan_nodes={}, indexed_nodes={}",
label,
pass,
scan_stats.edges_streamed,
indexed_stats.edges_streamed,
scan_stats.nodes_seen,
indexed_stats.nodes_seen
);
}
} else {
info!(
"Adjacency shadow skipped for {} {}; index is not built",
label, pass
);
}
scan_stats
}
}
};
let stats = Phase3StreamStats {
edges_streamed: connected_stats.edges_streamed,
nodes_seen: connected_stats.nodes_seen,
used_adjacency_index: connected_stats.used_adjacency_index,
peak_rss_mb,
elapsed_secs: start.elapsed().as_secs_f64(),
};
let lookup = if stats.used_adjacency_index {
"adjacency index"
} else {
"full scan"
};
match stats.peak_rss_mb {
Some(peak) => info!(
"Phase 3 {} for {} streamed {} connected edges across {} nodes using {} in {:.2}s; peak RSS {:.1} MB",
pass,
label,
stats.edges_streamed,
stats.nodes_seen,
lookup,
stats.elapsed_secs,
peak
),
None => info!(
"Phase 3 {} for {} streamed {} connected edges across {} nodes using {} in {:.2}s",
pass,
label,
stats.edges_streamed,
stats.nodes_seen,
lookup,
stats.elapsed_secs
),
}
Ok(stats)
}
fn discover_streamed_component_doc_ids(
matches_store: &MatchStore,
seed_doc_ids: &[Uuid],
mode: EdgeLookupMode,
label: &str,
) -> Result<(Vec<Uuid>, Phase3StreamStats)> {
let mut component_doc_ids = HashSet::new();
let stats = stream_connected_matches(
matches_store,
seed_doc_ids,
mode,
label,
"component discovery",
|record| {
component_doc_ids.insert(record.child_id);
component_doc_ids.insert(record.parent_id);
Ok(())
},
)?;
let mut ids: Vec<Uuid> = component_doc_ids.into_iter().collect();
ids.sort();
Ok((ids, stats))
}
fn resolve_transitivity_streamed(
matches_store: &MatchStore,
seed_doc_ids: &[Uuid],
mode: EdgeLookupMode,
label: &str,
preferred_parent_ids: &HashSet<Uuid>,
new_doc_set: &HashSet<Uuid>,
) -> Result<StreamedTransitivity> {
let mut uf = UnionFind::new();
let mut best_similarity: HashMap<Uuid, (Uuid, Uuid, f64, i32, f64)> = HashMap::new();
stream_connected_matches(
matches_store,
seed_doc_ids,
mode,
label,
"transitivity resolution",
|record| {
uf.make_set(record.child_id);
uf.make_set(record.parent_id);
uf.union_by_key(record.child_id, record.parent_id, |id| {
let priority = if preferred_parent_ids.contains(&id) {
0u8
} else if new_doc_set.contains(&id) {
2u8
} else {
1u8
};
(priority, id)
});
for doc_id in [record.child_id, record.parent_id] {
best_similarity
.entry(doc_id)
.and_modify(|e| {
if record.jaccard_similarity > e.2
|| (record.jaccard_similarity == e.2
&& (record.child_id, record.parent_id) < (e.0, e.1))
{
*e = (
record.child_id,
record.parent_id,
record.jaccard_similarity,
record.size_difference,
record.size_difference_pct,
);
}
})
.or_insert((
record.child_id,
record.parent_id,
record.jaccard_similarity,
record.size_difference,
record.size_difference_pct,
));
}
Ok(())
},
)?;
let mut assignments = Vec::new();
let mut parent_ids = HashSet::new();
let mut child_ids = HashSet::new();
for node in uf.nodes() {
let root = uf.find(node);
if root != node {
child_ids.insert(node);
parent_ids.insert(root);
let (_, _, similarity, size_diff, size_diff_pct) = best_similarity
.get(&node)
.copied()
.unwrap_or((Uuid::nil(), Uuid::nil(), 0.8, 0, 0.0));
assignments.push(DupeMatch {
child_id: node,
parent_id: root,
jaccard_similarity: similarity,
size_difference: size_diff,
size_difference_pct: size_diff_pct,
});
} else {
parent_ids.insert(node);
}
}
assignments.sort_by_key(|m| (m.child_id, m.parent_id));
Ok(StreamedTransitivity {
assignments,
parent_ids,
child_ids,
})
}
#[derive(Debug, Default)]
pub struct DedupeStats {
pub total_documents: usize,
pub duplicates_found: usize,
pub unique_parents: usize,
pub candidates_checked: usize,
pub duration_secs: f64,
pub skipped_short: usize,
pub skipped_boilerplate: usize,
}
pub fn load_junk_patterns<P: AsRef<Path>>(path: P) -> Result<Vec<String>> {
let content = std::fs::read_to_string(path.as_ref())
.with_context(|| format!("Failed to read junk patterns from {:?}", path.as_ref()))?;
let patterns: Vec<String> = content
.lines()
.map(|line| line.trim())
.filter(|line| !line.is_empty() && !line.starts_with('#'))
.map(|line| line.replace("\\n", "\n"))
.collect();
info!(
"Loaded {} junk patterns from {:?}",
patterns.len(),
path.as_ref()
);
Ok(patterns)
}
pub fn is_boilerplate(content: &str, custom_patterns: &[String]) -> bool {
let content_trimmed = content.trim();
if content_trimmed.is_empty() {
return true;
}
for pattern in custom_patterns {
if content.contains(pattern.as_str()) {
return true;
}
}
let lines: Vec<&str> = content_trimmed.lines().collect();
if lines.len() > 10 {
let avg_line_len: f64 = content_trimmed.len() as f64 / lines.len() as f64;
if avg_line_len < 20.0 && lines.len() > 20 {
return true;
}
}
false
}
fn take_pathological_batch_ids(
new_doc_ids: &mut Vec<Uuid>,
pathological_doc_ids: &HashSet<Uuid>,
) -> Vec<Uuid> {
let mut pathological_batch_ids = Vec::new();
new_doc_ids.retain(|id| {
if pathological_doc_ids.contains(id) {
pathological_batch_ids.push(*id);
false
} else {
true
}
});
pathological_batch_ids
}
fn clear_sidecar_files(dataset_dir: &Path) -> Result<()> {
for file_name in [
"lsh.redb",
"matches.redb",
"state.redb",
"filtered_parents.redb",
] {
let path = dataset_dir.join(file_name);
if path.exists() {
std::fs::remove_file(&path)
.with_context(|| format!("Failed to remove stale sidecar file {:?}", path))?;
info!("Removed stale sidecar file {:?}", path);
}
}
Ok(())
}
fn validate_dedupe_config(config: &DedupeConfig) -> Result<()> {
if !(0.0..=1.0).contains(&config.threshold) {
anyhow::bail!("threshold must be between 0.0 and 1.0");
}
if config.size_diff_threshold < 0.0 {
anyhow::bail!("size_diff_threshold must be non-negative");
}
if config.batch_size <= 0 {
anyhow::bail!("batch_size must be positive");
}
if config.num_workers == 0 {
anyhow::bail!("num_workers must be positive");
}
if config.min_content_length < 0 {
anyhow::bail!("min_content_length must be non-negative");
}
if config.max_matches_per_doc == Some(0) {
anyhow::bail!("max_matches_per_doc must be positive when set");
}
Ok(())
}
fn batch_size_usize(batch_size: i64) -> Result<usize> {
if batch_size <= 0 {
anyhow::bail!("batch_size must be positive");
}
usize::try_from(batch_size).context("batch_size does not fit in usize")
}
async fn flush_filtered_parents_pool(pool: &DbPool, buffer: &mut Vec<Uuid>) -> Result<()> {
if !buffer.is_empty() {
pool.mark_as_parents(buffer).await?;
buffer.clear();
}
Ok(())
}
async fn flush_filtered_parents_source<S: DocumentSource>(
source: &S,
buffer: &mut Vec<Uuid>,
) -> Result<()> {
if !buffer.is_empty() {
source.mark_as_parents(buffer).await?;
buffer.clear();
}
Ok(())
}
fn flush_filtered_parents_sidecar(dataset_dir: &Path, buffer: &mut Vec<Uuid>) -> Result<()> {
if !buffer.is_empty() {
let store = FilteredParentStore::open(dataset_dir.join("filtered_parents.redb"))?;
store.insert_batch(buffer)?;
buffer.clear();
}
Ok(())
}
fn assignments_requiring_write(
resolved_matches: &[DupeMatch],
new_doc_set: &HashSet<Uuid>,
existing_dupe_parents: &HashMap<Uuid, Uuid>,
) -> Vec<DupeMatch> {
resolved_matches
.iter()
.filter(|m| {
new_doc_set.contains(&m.child_id)
|| existing_dupe_parents.get(&m.child_id) != Some(&m.parent_id)
})
.cloned()
.collect()
}
fn child_marks_to_apply(
child_ids: &HashSet<Uuid>,
new_doc_set: &HashSet<Uuid>,
existing_parent_ids: &HashSet<Uuid>,
) -> Vec<Uuid> {
child_ids
.iter()
.filter(|id| new_doc_set.contains(id) || existing_parent_ids.contains(id))
.copied()
.collect()
}
fn parent_marks_to_apply(
parent_ids: &HashSet<Uuid>,
new_doc_ids: &[Uuid],
child_ids: &HashSet<Uuid>,
existing_dupe_parents: &HashMap<Uuid, Uuid>,
) -> Vec<Uuid> {
let mut ids: HashSet<Uuid> = new_doc_ids
.iter()
.filter(|id| !child_ids.contains(id))
.copied()
.collect();
ids.extend(
parent_ids
.iter()
.filter(|id| existing_dupe_parents.contains_key(id))
.copied(),
);
let mut ids: Vec<Uuid> = ids.into_iter().collect();
ids.sort();
ids
}
async fn flush_filtered_parents_db_run(
pool: &DbPool,
dataset_dir: &Path,
skip_db_write: bool,
buffer: &mut Vec<Uuid>,
) -> Result<()> {
if skip_db_write {
flush_filtered_parents_sidecar(dataset_dir, buffer)
} else {
flush_filtered_parents_pool(pool, buffer).await
}
}
async fn flush_filtered_parents_source_run<S: DocumentSource>(
source: &S,
dataset_dir: &Path,
write_to_source: bool,
buffer: &mut Vec<Uuid>,
) -> Result<()> {
if write_to_source {
flush_filtered_parents_source(source, buffer).await
} else {
flush_filtered_parents_sidecar(dataset_dir, buffer)
}
}
pub struct InMemoryDeduplicator {
config: DedupeConfig,
lsh: InMemoryLSH,
doc_sizes: HashMap<Uuid, i32>,
}
impl InMemoryDeduplicator {
pub fn new(config: DedupeConfig) -> Self {
Self {
config,
lsh: InMemoryLSH::new(),
doc_sizes: HashMap::new(),
}
}
pub fn index_documents(&mut self, documents: &[Document]) {
let seed = self.config.seed;
let pb = ProgressBar::new(documents.len() as u64);
pb.set_style(
ProgressStyle::default_bar()
.template("[{elapsed_precise}] {bar:40.cyan/blue} {pos}/{len} ({per_sec}) {msg}")
.unwrap(),
);
pb.set_message("Computing signatures...");
let signatures: Vec<(Uuid, Vec<u32>, i32)> = documents
.par_iter()
.map(|doc| {
let sig = compute_signature(&doc.content, seed);
pb.inc(1);
(doc.id, sig, doc.content_len)
})
.collect();
pb.finish_with_message("Signatures computed");
let pb = ProgressBar::new(signatures.len() as u64);
pb.set_style(
ProgressStyle::default_bar()
.template("[{elapsed_precise}] {bar:40.green/blue} {pos}/{len} Indexing...")
.unwrap(),
);
for (doc_id, signature, content_len) in signatures {
self.doc_sizes.insert(doc_id, content_len);
self.lsh.insert(doc_id, signature);
pb.inc(1);
}
pb.finish_with_message("Indexing complete");
}
pub fn find_duplicates(&self) -> HashMap<Uuid, Uuid> {
let matches = self.find_duplicates_with_scores();
let pairs: Vec<(Uuid, Uuid)> = matches.iter().map(|m| (m.child_id, m.parent_id)).collect();
build_parent_assignments(&pairs)
}
pub fn find_duplicates_with_scores(&self) -> Vec<DuplicateMatch> {
let doc_ids: Vec<Uuid> = self.lsh.doc_ids().copied().collect();
let total = doc_ids.len();
let pb = ProgressBar::new(total as u64);
pb.set_style(
ProgressStyle::default_bar()
.template("[{elapsed_precise}] {bar:40.yellow/blue} {pos}/{len} ({per_sec}) Finding duplicates...")
.unwrap(),
);
let candidates_checked = AtomicUsize::new(0);
let duplicates_found = AtomicUsize::new(0);
let matches: Vec<DuplicateMatch> = doc_ids
.par_iter()
.flat_map(|&doc_id| {
let Some(sig) = self.lsh.get_signature(&doc_id) else {
warn!("Missing signature for indexed document {}", doc_id);
pb.inc(1);
return Vec::new();
};
let doc_size = *self.doc_sizes.get(&doc_id).unwrap_or(&0);
let candidates = self.lsh.query(sig);
candidates_checked.fetch_add(candidates.len(), Ordering::Relaxed);
let mut pairs = Vec::new();
for &candidate_id in &candidates {
if candidate_id == doc_id {
continue;
}
if doc_id >= candidate_id {
continue;
}
let cand_size = *self.doc_sizes.get(&candidate_id).unwrap_or(&0);
if !size_within_threshold(doc_size, cand_size, self.config.size_diff_threshold)
{
continue;
}
let Some(cand_sig) = self.lsh.get_signature(&candidate_id) else {
warn!("Missing signature for candidate document {}", candidate_id);
continue;
};
let jaccard = jaccard_from_signatures(sig, cand_sig);
if jaccard >= self.config.threshold {
duplicates_found.fetch_add(1, Ordering::Relaxed);
let (child_id, child_size, parent_id, parent_size) =
if doc_size >= cand_size {
(doc_id, doc_size, candidate_id, cand_size)
} else {
(candidate_id, cand_size, doc_id, doc_size)
};
let size_diff = (child_size - parent_size).abs();
let larger_size = child_size.max(parent_size);
let size_diff_pct = if larger_size > 0 {
size_diff as f64 / larger_size as f64
} else {
0.0
};
pairs.push(DuplicateMatch {
child_id,
parent_id,
jaccard_similarity: jaccard,
size_difference: size_diff,
size_difference_pct: size_diff_pct,
});
}
}
pb.inc(1);
pairs
})
.collect();
pb.finish_with_message(format!(
"Found {} duplicates from {} candidates",
duplicates_found.load(Ordering::Relaxed),
candidates_checked.load(Ordering::Relaxed)
));
matches
}
}
#[derive(Debug, Clone)]
pub struct DuplicateMatch {
pub child_id: Uuid,
pub parent_id: Uuid,
pub jaccard_similarity: f64,
pub size_difference: i32,
pub size_difference_pct: f64,
}
fn size_within_threshold(size1: i32, size2: i32, threshold: f64) -> bool {
if size1 == 0 || size2 == 0 {
return true;
}
let (smaller, larger) = if size1 < size2 {
(size1, size2)
} else {
(size2, size1)
};
let diff = (larger - smaller) as f64 / larger as f64;
diff <= threshold
}
fn build_parent_assignments(pairs: &[(Uuid, Uuid)]) -> HashMap<Uuid, Uuid> {
UnionFind::from_pairs(pairs)
}
pub struct IndexBuilder {
config: DedupeConfig,
lsh: DiskLSH,
}
impl IndexBuilder {
pub fn new<P: AsRef<Path>>(config: DedupeConfig, db_path: P) -> Result<Self> {
validate_dedupe_config(&config)?;
let lsh = DiskLSH::open(db_path)?;
lsh.validate_or_initialize_metadata(config.seed)?;
Ok(Self { config, lsh })
}
pub fn index_documents(&self, documents: &[Document]) -> Result<()> {
let seed = self.config.seed;
let batch_size = 1000;
let pb = ProgressBar::new(documents.len() as u64);
pb.set_style(
ProgressStyle::default_bar()
.template(
"[{elapsed_precise}] {bar:40.cyan/blue} {pos}/{len} ({per_sec}) Indexing...",
)
.unwrap(),
);
for chunk in documents.chunks(batch_size) {
let entries: Vec<(Uuid, Vec<u32>, usize)> = chunk
.par_iter()
.map(|doc| {
let sig = compute_signature(&doc.content, seed);
(doc.id, sig, doc.content_len as usize)
})
.collect();
self.lsh.insert_batch(&entries)?;
pb.inc(chunk.len() as u64);
}
pb.finish_with_message("Indexing complete");
Ok(())
}
#[deprecated(
since = "0.2.4",
note = "Use disk_dedupe::run_disk_dedupe after building the index"
)]
pub fn find_duplicates(&self) -> Result<HashMap<Uuid, Uuid>> {
anyhow::bail!(
"IndexBuilder::find_duplicates has been disabled; use disk_dedupe::run_disk_dedupe after index_documents"
)
}
}
pub async fn run_dedupe(db_config: DbConfig, dedupe_config: DedupeConfig) -> Result<DedupeStats> {
validate_dedupe_config(&dedupe_config)?;
let start = std::time::Instant::now();
log_memory("run_dedupe start");
let source_name = db_config.source_name();
let table_name = db_config.table_name.clone();
info!("Connecting to database...");
let pool = DbPool::new(db_config).await?;
info!("PostgreSQL table: {}", table_name);
if source_name != table_name {
info!("PostgreSQL scope: {}", source_name);
}
let dataset_dir = dedupe_config.data_dir.join(&source_name);
std::fs::create_dir_all(&dataset_dir)?;
let lsh_path = dataset_dir.join("lsh.redb");
info!("Data directory: {:?}", dataset_dir);
info!("LSH index: {:?}", lsh_path);
if dedupe_config.fresh {
info!("Fresh mode enabled: clearing local sidecar files before rebuild");
clear_sidecar_files(&dataset_dir)?;
}
let junk_patterns_path = dataset_dir.join("junk_patterns.txt");
let junk_patterns: Vec<String> = if junk_patterns_path.exists() {
load_junk_patterns(&junk_patterns_path)?
} else {
info!(
"No junk_patterns.txt found in {:?}, skipping boilerplate filtering",
dataset_dir
);
Vec::new()
};
let total = if dedupe_config.process_all {
let total = pool.count_total().await?;
info!("Found {} total documents (processing ALL)", total);
total
} else {
let total = pool.count_unprocessed().await?;
info!("Found {} unprocessed documents", total);
total
};
if total == 0 {
return Ok(DedupeStats::default());
}
log_memory("Before Phase 1");
info!("");
info!("=== Phase 1: Incremental LSH Index Build ===");
let open_pb = ProgressBar::new_spinner();
open_pb.set_style(
ProgressStyle::default_spinner()
.template("{spinner:.green} [{elapsed_precise}] Opening LSH index: {msg}")
.unwrap(),
);
open_pb.set_message(lsh_path.display().to_string());
open_pb.enable_steady_tick(std::time::Duration::from_millis(100));
let open_start = std::time::Instant::now();
let lsh = DiskLSH::open(&lsh_path)?;
lsh.validate_or_initialize_metadata(dedupe_config.seed)?;
let existing_count = lsh.count()?;
let has_bands = lsh.has_bands()?;
open_pb.finish_with_message(format!(
"Opened in {:.1}s: {} signatures, has_bands: {}",
open_start.elapsed().as_secs_f64(),
existing_count,
has_bands
));
info!(
"Existing index: {} signatures, has_bands: {}",
existing_count, has_bands
);
if existing_count > 0 && !has_bands {
info!("Legacy index detected: building bands from existing signatures...");
let band_start = std::time::Instant::now();
let band_count = lsh.build_bands_from_signatures()?;
info!(
"Built bands for {} docs in {:.2}s",
band_count,
band_start.elapsed().as_secs_f64()
);
}
info!("");
info!("=== Phase 1a: Finding documents not yet indexed ===");
let seed = dedupe_config.seed;
let min_content_len = dedupe_config.min_content_length;
let mut new_doc_ids: Vec<Uuid> = Vec::new();
let mut short_doc_count = 0usize;
let mut boilerplate_doc_count = 0usize;
let filtered_parent_batch_size = (dedupe_config.batch_size as usize).max(1);
let mut filtered_parent_buffer: Vec<Uuid> =
Vec::with_capacity(filtered_parent_batch_size.min(50_000));
let pb = ProgressBar::new(total as u64);
pb.set_style(
ProgressStyle::default_bar()
.template("[{elapsed_precise}] {bar:40.cyan/blue} {pos}/{len} ({per_sec}) Phase 1a: Checking & indexing new docs... ETA: {eta}")
.unwrap(),
);
let mut last_id: Option<Uuid> = None;
let index_batch_size = 1000;
let mut pending_entries: Vec<(Uuid, Vec<u32>, usize)> = Vec::with_capacity(index_batch_size);
let mut processed_count = 0u64;
loop {
let unprocessed_ids = if dedupe_config.process_all {
let docs = pool
.fetch_chunk_after_all(last_id, dedupe_config.batch_size)
.await?;
if docs.is_empty() {
break;
}
last_id = docs.last().map(|d| d.id);
for doc in docs {
if is_boilerplate(&doc.content, &junk_patterns) {
boilerplate_doc_count += 1;
filtered_parent_buffer.push(doc.id);
if filtered_parent_buffer.len() >= filtered_parent_batch_size {
flush_filtered_parents_db_run(
&pool,
&dataset_dir,
dedupe_config.skip_db_write,
&mut filtered_parent_buffer,
)
.await?;
}
continue;
}
if doc.content.len() as i32 <= min_content_len {
short_doc_count += 1;
filtered_parent_buffer.push(doc.id);
if filtered_parent_buffer.len() >= filtered_parent_batch_size {
flush_filtered_parents_db_run(
&pool,
&dataset_dir,
dedupe_config.skip_db_write,
&mut filtered_parent_buffer,
)
.await?;
}
continue;
}
if existing_count > 0 && lsh.has_document(&doc.id)? {
continue;
}
let sig = compute_signature(&doc.content, seed);
pending_entries.push((doc.id, sig, doc.content.len()));
new_doc_ids.push(doc.id);
if pending_entries.len() >= index_batch_size {
lsh.insert_batch(&pending_entries)?;
pending_entries.clear();
}
}
processed_count += dedupe_config.batch_size as u64;
pb.set_position(processed_count.min(total as u64));
continue;
} else {
pool.fetch_unprocessed_ids_after(last_id, dedupe_config.batch_size)
.await?
};
if unprocessed_ids.is_empty() {
break;
}
last_id = unprocessed_ids.last().copied();
processed_count += unprocessed_ids.len() as u64;
pb.set_position(processed_count.min(total as u64));
let mut ids_to_fetch: Vec<Uuid> = Vec::new();
for id in &unprocessed_ids {
if existing_count > 0 && lsh.has_document(id)? {
new_doc_ids.push(*id);
continue;
}
ids_to_fetch.push(*id);
}
if ids_to_fetch.is_empty() {
continue;
}
let docs = pool.fetch_documents_by_ids(&ids_to_fetch).await?;
for doc in docs {
if is_boilerplate(&doc.content, &junk_patterns) {
boilerplate_doc_count += 1;
filtered_parent_buffer.push(doc.id);
if filtered_parent_buffer.len() >= filtered_parent_batch_size {
flush_filtered_parents_db_run(
&pool,
&dataset_dir,
dedupe_config.skip_db_write,
&mut filtered_parent_buffer,
)
.await?;
}
continue;
}
if doc.content.len() as i32 <= min_content_len {
short_doc_count += 1;
filtered_parent_buffer.push(doc.id);
if filtered_parent_buffer.len() >= filtered_parent_batch_size {
flush_filtered_parents_db_run(
&pool,
&dataset_dir,
dedupe_config.skip_db_write,
&mut filtered_parent_buffer,
)
.await?;
}
continue;
}
let sig = compute_signature(&doc.content, seed);
pending_entries.push((doc.id, sig, doc.content.len()));
new_doc_ids.push(doc.id);
if pending_entries.len() >= index_batch_size {
lsh.insert_batch(&pending_entries)?;
pending_entries.clear();
}
}
}
if !pending_entries.is_empty() {
lsh.insert_batch(&pending_entries)?;
}
flush_filtered_parents_db_run(
&pool,
&dataset_dir,
dedupe_config.skip_db_write,
&mut filtered_parent_buffer,
)
.await?;
pb.finish_with_message("Phase 1a complete");
if short_doc_count > 0 || boilerplate_doc_count > 0 {
if short_doc_count > 0 {
info!(
"Skipped {} documents with content <= {} UTF-8 bytes",
short_doc_count, min_content_len
);
}
if boilerplate_doc_count > 0 {
info!(
"Skipped {} boilerplate/junk documents",
boilerplate_doc_count
);
}
}
info!("Phase 1 complete: indexed {} new docs.", new_doc_ids.len());
log_memory("After Phase 1");
let min_bucket_size = 1000; let min_bands = 14;
let pathological_doc_ids =
match crate::cleanup::run_phase_1_5(&lsh, &dataset_dir, min_bucket_size, min_bands) {
Ok(ids) => ids,
Err(e) => {
warn!(
"Failed to detect pathological clusters: {}. Continuing anyway.",
e
);
HashSet::new()
}
};
let original_count = new_doc_ids.len();
let pathological_batch_doc_ids =
take_pathological_batch_ids(&mut new_doc_ids, &pathological_doc_ids);
if !pathological_doc_ids.is_empty() {
info!(
"Detected {} pathological docs in index; {} are in the current batch",
pathological_doc_ids.len(),
pathological_batch_doc_ids.len()
);
}
if original_count != new_doc_ids.len() {
info!(
"Filtered {} pathological docs, {} remaining for Phase 2",
original_count - new_doc_ids.len(),
new_doc_ids.len()
);
}
info!("");
if new_doc_ids.is_empty() && pathological_batch_doc_ids.is_empty() {
info!("No new documents to dedupe - skipping Phase 2 and 3.");
return Ok(DedupeStats {
total_documents: total as usize,
unique_parents: short_doc_count + boilerplate_doc_count,
skipped_short: short_doc_count,
skipped_boilerplate: boilerplate_doc_count,
..DedupeStats::default()
});
}
let skip_phase2 = new_doc_ids.is_empty();
log_memory("Before Phase 2");
let (phase3_seed_doc_ids, candidates_checked) = if skip_phase2 {
info!("=== Phase 2: Skipped (only pathological docs) ===");
(pathological_batch_doc_ids.clone(), 0)
} else {
if !dedupe_config.disk_phase2 {
warn!("In-memory Phase 2 has been removed; using disk-based Phase 2");
}
info!("=== Phase 2: Finding Duplicates (DISK-BASED parallel) ===");
drop(lsh); info!("Released LSH index lock for Phase 2");
let disk_stats = crate::disk_dedupe::run_disk_dedupe_with_options(
&lsh_path,
&dataset_dir,
crate::disk_dedupe::DiskDedupeRunOptions {
num_workers: dedupe_config.num_workers,
threshold: dedupe_config.threshold,
size_diff_threshold: dedupe_config.size_diff_threshold,
fresh: false,
new_doc_ids: Some(new_doc_ids.clone()),
max_matches_per_doc: dedupe_config.max_matches_per_doc,
},
)?;
(new_doc_ids.clone(), disk_stats.candidates_checked)
};
info!(
"Phase 2 complete: checked {} candidates. Phase 3 will stream connected duplicate edges.",
candidates_checked
);
log_memory("After Phase 2");
if !pathological_batch_doc_ids.is_empty() {
new_doc_ids.extend(pathological_batch_doc_ids.iter().copied());
info!(
"Added {} pathological docs from current batch for Phase 3 sync",
pathological_batch_doc_ids.len()
);
}
let phase3_outcome = if !dedupe_config.skip_db_write {
log_memory("Before Phase 3");
info!("");
info!("=== Phase 3: Syncing to PostgreSQL ===");
let matches_store = crate::storage::MatchStore::open(dataset_dir.join("matches.redb"))?;
let outcome = perform_incremental_sync_streaming(
&pool,
&matches_store,
&phase3_seed_doc_ids,
&new_doc_ids,
dedupe_config.edge_lookup_mode,
"database incremental batch",
dedupe_config.batch_size,
)
.await?;
log_memory("After Phase 3");
outcome
} else {
info!("");
info!("=== Skipping PostgreSQL sync (--skip-db-write) ===");
Phase3SyncOutcome::default()
};
let duration = start.elapsed();
let final_parents = new_doc_ids
.len()
.saturating_sub(phase3_outcome.new_children)
+ short_doc_count
+ boilerplate_doc_count;
log_memory("run_dedupe end");
Ok(DedupeStats {
total_documents: total as usize,
duplicates_found: usize::try_from(phase3_outcome.connected_edges).unwrap_or(usize::MAX),
unique_parents: final_parents,
candidates_checked,
duration_secs: duration.as_secs_f64(),
skipped_short: short_doc_count,
skipped_boilerplate: boilerplate_doc_count,
})
}
pub async fn run_dedupe_with_source<S: DocumentSource>(
source: &S,
dedupe_config: DedupeConfig,
source_name: Option<&str>,
) -> Result<DedupeStats> {
validate_dedupe_config(&dedupe_config)?;
let start = std::time::Instant::now();
log_memory("run_dedupe_with_source start");
let dataset_name = match source_name {
Some(name) => name.to_string(),
None => source.source_name().await?,
};
info!("Source: {}", dataset_name);
let dataset_dir = dedupe_config.data_dir.join(&dataset_name);
std::fs::create_dir_all(&dataset_dir)?;
let lsh_path = dataset_dir.join("lsh.redb");
info!("Data directory: {:?}", dataset_dir);
info!("LSH index: {:?}", lsh_path);
if dedupe_config.fresh {
info!("Fresh mode enabled: clearing local sidecar files before rebuild");
clear_sidecar_files(&dataset_dir)?;
}
let junk_patterns_path = dataset_dir.join("junk_patterns.txt");
let junk_patterns: Vec<String> = if junk_patterns_path.exists() {
load_junk_patterns(&junk_patterns_path)?
} else {
info!(
"No junk_patterns.txt found in {:?}, skipping boilerplate filtering",
dataset_dir
);
Vec::new()
};
let total = if dedupe_config.process_all {
let total = source.count_total().await?;
info!("Found {} total documents (processing ALL)", total);
total
} else {
let total = source.count_unprocessed().await?;
info!("Found {} unprocessed documents", total);
total
};
if total == 0 {
return Ok(DedupeStats::default());
}
log_memory("Before Phase 1");
info!("");
info!("=== Phase 1: Incremental LSH Index Build ===");
let open_pb = ProgressBar::new_spinner();
open_pb.set_style(
ProgressStyle::default_spinner()
.template("{spinner:.green} [{elapsed_precise}] Opening LSH index: {msg}")
.unwrap(),
);
open_pb.set_message(lsh_path.display().to_string());
open_pb.enable_steady_tick(std::time::Duration::from_millis(100));
let open_start = std::time::Instant::now();
let lsh = DiskLSH::open(&lsh_path)?;
lsh.validate_or_initialize_metadata(dedupe_config.seed)?;
let existing_count = lsh.count()?;
let has_bands = lsh.has_bands()?;
open_pb.finish_with_message(format!(
"Opened in {:.1}s: {} signatures, has_bands: {}",
open_start.elapsed().as_secs_f64(),
existing_count,
has_bands
));
info!(
"Existing index: {} signatures, has_bands: {}",
existing_count, has_bands
);
if existing_count > 0 && !has_bands {
info!("Legacy index detected: building bands from existing signatures...");
let band_start = std::time::Instant::now();
let band_count = lsh.build_bands_from_signatures()?;
info!(
"Built bands for {} docs in {:.2}s",
band_count,
band_start.elapsed().as_secs_f64()
);
}
info!("");
info!("=== Phase 1a: Finding documents not yet indexed ===");
let seed = dedupe_config.seed;
let min_content_len = dedupe_config.min_content_length;
let mut new_doc_ids: Vec<Uuid> = Vec::new();
let mut short_doc_count = 0usize;
let mut boilerplate_doc_count = 0usize;
let mark_filtered_parents = !dedupe_config.skip_db_write && source.supports_write();
let filtered_parent_batch_size = (dedupe_config.batch_size as usize).max(1);
let mut filtered_parent_buffer: Vec<Uuid> =
Vec::with_capacity(filtered_parent_batch_size.min(50_000));
let pb = ProgressBar::new(total as u64);
pb.set_style(
ProgressStyle::default_bar()
.template("[{elapsed_precise}] {bar:40.cyan/blue} {pos}/{len} ({per_sec}) Phase 1a: Checking & indexing new docs... ETA: {eta}")
.unwrap(),
);
let mut last_id: Option<Uuid> = None;
let index_batch_size = 1000;
let mut pending_entries: Vec<(Uuid, Vec<u32>, usize)> = Vec::with_capacity(index_batch_size);
let mut processed_count = 0u64;
loop {
let docs = if dedupe_config.process_all {
source
.fetch_all_after(last_id, dedupe_config.batch_size)
.await?
} else {
let ids = source
.fetch_unprocessed_ids_after(last_id, dedupe_config.batch_size)
.await?;
if ids.is_empty() {
break;
}
last_id = ids.last().copied();
source.fetch_by_ids(&ids).await?
};
if docs.is_empty() {
break;
}
if dedupe_config.process_all {
last_id = docs.last().map(|d| d.id);
}
for doc in docs {
if is_boilerplate(&doc.content, &junk_patterns) {
boilerplate_doc_count += 1;
filtered_parent_buffer.push(doc.id);
if filtered_parent_buffer.len() >= filtered_parent_batch_size {
flush_filtered_parents_source_run(
source,
&dataset_dir,
mark_filtered_parents,
&mut filtered_parent_buffer,
)
.await?;
}
continue;
}
if doc.content.len() as i32 <= min_content_len {
short_doc_count += 1;
filtered_parent_buffer.push(doc.id);
if filtered_parent_buffer.len() >= filtered_parent_batch_size {
flush_filtered_parents_source_run(
source,
&dataset_dir,
mark_filtered_parents,
&mut filtered_parent_buffer,
)
.await?;
}
continue;
}
if existing_count > 0 && lsh.has_document(&doc.id)? {
if !dedupe_config.process_all {
new_doc_ids.push(doc.id);
}
continue;
}
let sig = compute_signature(&doc.content, seed);
pending_entries.push((doc.id, sig, doc.content.len()));
new_doc_ids.push(doc.id);
if pending_entries.len() >= index_batch_size {
lsh.insert_batch(&pending_entries)?;
pending_entries.clear();
}
}
processed_count += dedupe_config.batch_size as u64;
pb.set_position(processed_count.min(total as u64));
}
if !pending_entries.is_empty() {
lsh.insert_batch(&pending_entries)?;
}
flush_filtered_parents_source_run(
source,
&dataset_dir,
mark_filtered_parents,
&mut filtered_parent_buffer,
)
.await?;
pb.finish_with_message("Phase 1a complete");
if short_doc_count > 0 || boilerplate_doc_count > 0 {
if short_doc_count > 0 {
info!(
"Skipped {} documents with content <= {} UTF-8 bytes",
short_doc_count, min_content_len
);
}
if boilerplate_doc_count > 0 {
info!(
"Skipped {} boilerplate/junk documents",
boilerplate_doc_count
);
}
}
info!("Phase 1 complete: indexed {} new docs.", new_doc_ids.len());
log_memory("After Phase 1");
let min_bucket_size = 1000;
let min_bands = 14;
let pathological_doc_ids =
match crate::cleanup::run_phase_1_5(&lsh, &dataset_dir, min_bucket_size, min_bands) {
Ok(ids) => ids,
Err(e) => {
warn!(
"Failed to detect pathological clusters: {}. Continuing anyway.",
e
);
HashSet::new()
}
};
let original_count = new_doc_ids.len();
let pathological_batch_doc_ids =
take_pathological_batch_ids(&mut new_doc_ids, &pathological_doc_ids);
if !pathological_doc_ids.is_empty() {
info!(
"Detected {} pathological docs in index; {} are in the current batch",
pathological_doc_ids.len(),
pathological_batch_doc_ids.len()
);
}
if original_count != new_doc_ids.len() {
info!(
"Filtered {} pathological docs, {} remaining for Phase 2",
original_count - new_doc_ids.len(),
new_doc_ids.len()
);
}
info!("");
if new_doc_ids.is_empty() && pathological_batch_doc_ids.is_empty() {
info!("No new documents to dedupe - skipping Phase 2 and 3.");
return Ok(DedupeStats {
total_documents: total as usize,
unique_parents: short_doc_count + boilerplate_doc_count,
skipped_short: short_doc_count,
skipped_boilerplate: boilerplate_doc_count,
..DedupeStats::default()
});
}
let skip_phase2 = new_doc_ids.is_empty();
log_memory("Before Phase 2");
let (phase3_seed_doc_ids, candidates_checked) = if skip_phase2 {
info!("=== Phase 2: Skipped (only pathological docs) ===");
(pathological_batch_doc_ids.clone(), 0)
} else {
if !dedupe_config.disk_phase2 {
warn!("In-memory Phase 2 has been removed; using disk-based Phase 2");
}
info!("=== Phase 2: Finding Duplicates (DISK-BASED parallel) ===");
drop(lsh); info!("Released LSH index lock for Phase 2");
let disk_stats = crate::disk_dedupe::run_disk_dedupe_with_options(
&lsh_path,
&dataset_dir,
crate::disk_dedupe::DiskDedupeRunOptions {
num_workers: dedupe_config.num_workers,
threshold: dedupe_config.threshold,
size_diff_threshold: dedupe_config.size_diff_threshold,
fresh: false,
new_doc_ids: Some(new_doc_ids.clone()),
max_matches_per_doc: dedupe_config.max_matches_per_doc,
},
)?;
(new_doc_ids.clone(), disk_stats.candidates_checked)
};
info!(
"Phase 2 complete: checked {} candidates. Phase 3 will stream connected duplicate edges.",
candidates_checked
);
log_memory("After Phase 2");
if !pathological_batch_doc_ids.is_empty() {
new_doc_ids.extend(pathological_batch_doc_ids.iter().copied());
info!(
"Added {} pathological docs from current batch for Phase 3 sync",
pathological_batch_doc_ids.len()
);
}
let phase3_outcome = if !dedupe_config.skip_db_write && source.supports_write() {
log_memory("Before Phase 3");
info!("");
info!("=== Phase 3: Syncing to Data Source ===");
let matches_store = crate::storage::MatchStore::open(dataset_dir.join("matches.redb"))?;
let outcome = perform_incremental_sync_generic_streaming(
source,
&matches_store,
&phase3_seed_doc_ids,
&new_doc_ids,
dedupe_config.edge_lookup_mode,
"source incremental batch",
dedupe_config.batch_size,
)
.await?;
log_memory("After Phase 3");
outcome
} else if dedupe_config.skip_db_write {
info!("");
info!("=== Skipping sync (--skip-db-write) ===");
Phase3SyncOutcome::default()
} else {
info!("");
info!("=== Skipping sync (source does not support write) ===");
Phase3SyncOutcome::default()
};
let duration = start.elapsed();
let final_parents = new_doc_ids
.len()
.saturating_sub(phase3_outcome.new_children)
+ short_doc_count
+ boilerplate_doc_count;
log_memory("run_dedupe_with_source end");
Ok(DedupeStats {
total_documents: total as usize,
duplicates_found: usize::try_from(phase3_outcome.connected_edges).unwrap_or(usize::MAX),
unique_parents: final_parents,
candidates_checked,
duration_secs: duration.as_secs_f64(),
skipped_short: short_doc_count,
skipped_boilerplate: boilerplate_doc_count,
})
}
async fn perform_incremental_sync_generic_streaming<S: DocumentSource>(
source: &S,
matches_store: &MatchStore,
phase3_seed_doc_ids: &[Uuid],
new_doc_ids: &[Uuid],
edge_lookup_mode: EdgeLookupMode,
label: &str,
batch_size: i64,
) -> Result<Phase3SyncOutcome> {
let batch_size = batch_size_usize(batch_size)?;
info!(
"Starting streaming incremental sync for {} new documents...",
new_doc_ids.len()
);
let new_doc_set: HashSet<Uuid> = new_doc_ids.iter().copied().collect();
let (component_doc_ids, discovery_stats) = discover_streamed_component_doc_ids(
matches_store,
phase3_seed_doc_ids,
edge_lookup_mode,
label,
)?;
let existing_parent_ids = source.fetch_existing_parent_ids(&component_doc_ids).await?;
let existing_dupe_parents = source
.fetch_existing_dupe_parents(&component_doc_ids)
.await?;
let StreamedTransitivity {
assignments: resolved_matches,
parent_ids: all_parent_ids,
child_ids: all_child_ids,
} = resolve_transitivity_streamed(
matches_store,
phase3_seed_doc_ids,
edge_lookup_mode,
label,
&existing_parent_ids,
&new_doc_set,
)?;
let assignments_to_write =
assignments_requiring_write(&resolved_matches, &new_doc_set, &existing_dupe_parents);
info!(
"Resolved {} total matches from {} streamed component edges; {} require source writes.",
resolved_matches.len(),
discovery_stats.edges_streamed,
assignments_to_write.len()
);
if !assignments_to_write.is_empty() {
info!(
"Writing {} new assignments to source...",
assignments_to_write.len()
);
let source_matches: Vec<SourceDupeMatch> = assignments_to_write
.iter()
.map(|m| SourceDupeMatch {
child_id: m.child_id,
parent_id: m.parent_id,
jaccard_similarity: m.jaccard_similarity,
size_difference: m.size_difference,
size_difference_pct: m.size_difference_pct,
})
.collect();
for chunk in source_matches.chunks(batch_size) {
source.write_dupes(chunk).await?;
}
}
let new_children = all_child_ids
.iter()
.filter(|id| new_doc_set.contains(id))
.count();
if source.tracks_state() {
let child_ids_to_mark =
child_marks_to_apply(&all_child_ids, &new_doc_set, &existing_parent_ids);
if !child_ids_to_mark.is_empty() {
info!(
"Marking {} documents as children...",
child_ids_to_mark.len()
);
for chunk in child_ids_to_mark.chunks(batch_size) {
source.mark_as_children(chunk).await?;
}
}
let parent_ids_to_mark = parent_marks_to_apply(
&all_parent_ids,
new_doc_ids,
&all_child_ids,
&existing_dupe_parents,
);
if !parent_ids_to_mark.is_empty() {
info!(
"Marking {} documents as parents...",
parent_ids_to_mark.len()
);
for chunk in parent_ids_to_mark.chunks(batch_size) {
source.mark_as_parents(chunk).await?;
}
}
}
info!("Streaming incremental sync complete.");
Ok(Phase3SyncOutcome {
connected_edges: discovery_stats.edges_streamed,
new_children,
})
}
async fn perform_incremental_sync_streaming(
pool: &DbPool,
matches_store: &MatchStore,
phase3_seed_doc_ids: &[Uuid],
new_doc_ids: &[Uuid],
edge_lookup_mode: EdgeLookupMode,
label: &str,
batch_size: i64,
) -> Result<Phase3SyncOutcome> {
let batch_size = batch_size_usize(batch_size)?;
info!(
"Starting streaming incremental sync for {} new documents...",
new_doc_ids.len()
);
let new_doc_set: HashSet<Uuid> = new_doc_ids.iter().copied().collect();
let (component_doc_ids, discovery_stats) = discover_streamed_component_doc_ids(
matches_store,
phase3_seed_doc_ids,
edge_lookup_mode,
label,
)?;
let existing_parent_ids = pool.fetch_parent_ids_by_doc_ids(&component_doc_ids).await?;
let existing_dupe_parents = pool
.fetch_dupe_parents_by_child_ids(&component_doc_ids)
.await?;
let StreamedTransitivity {
assignments: resolved_matches,
parent_ids: all_parent_ids,
child_ids: all_child_ids,
} = resolve_transitivity_streamed(
matches_store,
phase3_seed_doc_ids,
edge_lookup_mode,
label,
&existing_parent_ids,
&new_doc_set,
)?;
let assignments_to_write =
assignments_requiring_write(&resolved_matches, &new_doc_set, &existing_dupe_parents);
info!(
"Resolved {} total matches from {} streamed component edges; {} require database writes.",
resolved_matches.len(),
discovery_stats.edges_streamed,
assignments_to_write.len()
);
if !assignments_to_write.is_empty() {
info!(
"Writing {} new assignments to database...",
assignments_to_write.len()
);
for chunk in assignments_to_write.chunks(batch_size) {
pool.write_dupes(chunk).await?;
}
}
let child_ids_to_mark =
child_marks_to_apply(&all_child_ids, &new_doc_set, &existing_parent_ids);
if !child_ids_to_mark.is_empty() {
info!(
"Marking {} documents as children...",
child_ids_to_mark.len()
);
for chunk in child_ids_to_mark.chunks(batch_size) {
pool.mark_as_children(chunk).await?;
}
}
let parent_ids_to_mark = parent_marks_to_apply(
&all_parent_ids,
new_doc_ids,
&all_child_ids,
&existing_dupe_parents,
);
if !parent_ids_to_mark.is_empty() {
info!(
"Marking {} documents as parents...",
parent_ids_to_mark.len()
);
for chunk in parent_ids_to_mark.chunks(batch_size) {
pool.mark_as_parents(chunk).await?;
}
}
let new_children = all_child_ids
.iter()
.filter(|id| new_doc_set.contains(id))
.count();
info!("Streaming incremental sync complete.");
Ok(Phase3SyncOutcome {
connected_edges: discovery_stats.edges_streamed,
new_children,
})
}
pub fn resolve_transitivity(
matches: &[MatchRecord],
) -> (Vec<DupeMatch>, HashSet<Uuid>, HashSet<Uuid>) {
resolve_transitivity_with_preferred_roots(matches, &HashSet::new(), None)
}
fn resolve_transitivity_with_preferred_roots(
matches: &[MatchRecord],
preferred_parent_ids: &HashSet<Uuid>,
new_doc_ids: Option<&HashSet<Uuid>>,
) -> (Vec<DupeMatch>, HashSet<Uuid>, HashSet<Uuid>) {
use crate::union_find::UnionFind;
if matches.is_empty() {
return (vec![], HashSet::new(), HashSet::new());
}
let mut uf = UnionFind::new();
let mut explicit_parent_ids: HashSet<Uuid> = HashSet::new();
for m in matches {
if m.child_id == m.parent_id {
explicit_parent_ids.insert(m.child_id);
}
}
let mut root_preferences = preferred_parent_ids.clone();
root_preferences.extend(explicit_parent_ids.iter().copied());
let mut best_similarity: HashMap<Uuid, (Uuid, Uuid, f64, i32, f64)> = HashMap::new();
let mut update_best = |doc_id: Uuid, m: &MatchRecord| {
best_similarity
.entry(doc_id)
.and_modify(|e| {
if m.jaccard_similarity > e.2
|| (m.jaccard_similarity == e.2 && (m.child_id, m.parent_id) < (e.0, e.1))
{
*e = (
m.child_id,
m.parent_id,
m.jaccard_similarity,
m.size_difference,
m.size_difference_pct,
);
}
})
.or_insert((
m.child_id,
m.parent_id,
m.jaccard_similarity,
m.size_difference,
m.size_difference_pct,
));
};
for m in matches {
if m.child_id == m.parent_id {
continue;
}
uf.make_set(m.child_id);
uf.make_set(m.parent_id);
uf.union_by_key(m.child_id, m.parent_id, |id| {
let priority = if root_preferences.contains(&id) {
0u8
} else if new_doc_ids.is_some_and(|ids| ids.contains(&id)) {
2u8
} else {
1u8
};
(priority, id)
});
update_best(m.child_id, m);
update_best(m.parent_id, m);
}
let mut assignments: Vec<DupeMatch> = Vec::new();
let mut parent_ids: HashSet<Uuid> = explicit_parent_ids;
let mut child_ids: HashSet<Uuid> = HashSet::new();
for node in uf.nodes() {
let root = uf.find(node);
if root != node {
child_ids.insert(node);
parent_ids.insert(root);
let (_, _, similarity, size_diff, size_diff_pct) = best_similarity
.get(&node)
.copied()
.unwrap_or((Uuid::nil(), Uuid::nil(), 0.8, 0, 0.0));
assignments.push(DupeMatch {
child_id: node,
parent_id: root, jaccard_similarity: similarity,
size_difference: size_diff,
size_difference_pct: size_diff_pct,
});
} else {
parent_ids.insert(node);
}
}
assignments.sort_by_key(|m| (m.child_id, m.parent_id));
(assignments, parent_ids, child_ids)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::tokenizer::tokenize;
#[test]
fn test_size_within_threshold() {
assert!(size_within_threshold(100, 110, 0.3)); assert!(size_within_threshold(100, 130, 0.3)); assert!(!size_within_threshold(100, 150, 0.3)); }
#[test]
fn test_build_parent_assignments() {
let id1 = Uuid::parse_str("00000000-0000-0000-0000-000000000001").unwrap();
let id2 = Uuid::parse_str("00000000-0000-0000-0000-000000000002").unwrap();
let id3 = Uuid::parse_str("00000000-0000-0000-0000-000000000003").unwrap();
let pairs = vec![(id2, id1), (id3, id2)];
let assignments = build_parent_assignments(&pairs);
assert_eq!(assignments.get(&id2), Some(&id1));
assert_eq!(assignments.get(&id3), Some(&id1));
assert!(!assignments.contains_key(&id1)); }
#[test]
fn test_tokenize() {
let text = "新华社北京七月四日电国务院发布新的通知";
let tokens = tokenize(text);
assert!(
tokens.len() > 10,
"raw Chinese should produce overlapping character n-grams"
);
assert_eq!(tokens[0], "新华社");
}
#[test]
fn test_take_pathological_batch_ids_only_returns_current_batch() {
let id1 = Uuid::parse_str("00000000-0000-0000-0000-000000000001").unwrap();
let id2 = Uuid::parse_str("00000000-0000-0000-0000-000000000002").unwrap();
let id3 = Uuid::parse_str("00000000-0000-0000-0000-000000000003").unwrap();
let historical_id = Uuid::parse_str("00000000-0000-0000-0000-000000000004").unwrap();
let mut batch_ids = vec![id1, id2, id3];
let pathological_ids = [id2, historical_id].into_iter().collect();
let pathological_batch_ids = take_pathological_batch_ids(&mut batch_ids, &pathological_ids);
assert_eq!(batch_ids, vec![id1, id3]);
assert_eq!(pathological_batch_ids, vec![id2]);
}
#[test]
fn test_resolve_transitivity_returns_sorted_assignments() {
let root = Uuid::parse_str("00000000-0000-0000-0000-000000000001").unwrap();
let child_a = Uuid::parse_str("00000000-0000-0000-0000-000000000003").unwrap();
let child_b = Uuid::parse_str("00000000-0000-0000-0000-000000000002").unwrap();
let matches = vec![
MatchRecord {
child_id: child_a,
parent_id: root,
jaccard_similarity: 0.9,
size_difference: 1,
size_difference_pct: 0.01,
},
MatchRecord {
child_id: child_b,
parent_id: root,
jaccard_similarity: 0.9,
size_difference: 1,
size_difference_pct: 0.01,
},
];
let (resolved, _, _) = resolve_transitivity(&matches);
assert_eq!(resolved.len(), 2);
assert_eq!(resolved[0].child_id, child_b);
assert_eq!(resolved[1].child_id, child_a);
}
#[test]
fn test_incremental_transitivity_prefers_existing_parent_over_lower_new_uuid() {
let new_lower_uuid = Uuid::parse_str("00000000-0000-0000-0000-000000000001").unwrap();
let existing_parent = Uuid::parse_str("00000000-0000-0000-0000-000000000010").unwrap();
let existing_child = Uuid::parse_str("00000000-0000-0000-0000-000000000011").unwrap();
let matches = vec![
MatchRecord {
child_id: existing_child,
parent_id: existing_parent,
jaccard_similarity: 0.95,
size_difference: 1,
size_difference_pct: 0.01,
},
MatchRecord {
child_id: existing_parent,
parent_id: new_lower_uuid,
jaccard_similarity: 0.90,
size_difference: 2,
size_difference_pct: 0.02,
},
];
let preferred_parent_ids = [existing_parent].into_iter().collect();
let new_doc_ids = [new_lower_uuid].into_iter().collect();
let (resolved, parent_ids, child_ids) = resolve_transitivity_with_preferred_roots(
&matches,
&preferred_parent_ids,
Some(&new_doc_ids),
);
assert!(parent_ids.contains(&existing_parent));
assert!(!child_ids.contains(&existing_parent));
assert!(resolved
.iter()
.any(|m| { m.child_id == new_lower_uuid && m.parent_id == existing_parent }));
assert!(resolved
.iter()
.any(|m| { m.child_id == existing_child && m.parent_id == existing_parent }));
}
#[test]
fn test_transitivity_metrics_are_tracked_for_both_edge_endpoints() {
let raw_child_preferred_root =
Uuid::parse_str("00000000-0000-0000-0000-000000000010").unwrap();
let raw_parent_later_child =
Uuid::parse_str("00000000-0000-0000-0000-000000000001").unwrap();
let matches = vec![MatchRecord {
child_id: raw_child_preferred_root,
parent_id: raw_parent_later_child,
jaccard_similarity: 0.93,
size_difference: 7,
size_difference_pct: 0.07,
}];
let preferred_parent_ids = [raw_child_preferred_root].into_iter().collect();
let (resolved, _, _) =
resolve_transitivity_with_preferred_roots(&matches, &preferred_parent_ids, None);
assert_eq!(resolved.len(), 1);
assert_eq!(resolved[0].child_id, raw_parent_later_child);
assert_eq!(resolved[0].parent_id, raw_child_preferred_root);
assert!((resolved[0].jaccard_similarity - 0.93).abs() < 0.001);
assert_eq!(resolved[0].size_difference, 7);
assert!((resolved[0].size_difference_pct - 0.07).abs() < 0.001);
}
#[test]
fn test_transitivity_equal_jaccard_tie_uses_lowest_edge_key() {
let root = Uuid::parse_str("00000000-0000-0000-0000-000000000001").unwrap();
let child = Uuid::parse_str("00000000-0000-0000-0000-000000000002").unwrap();
let higher_edge_key = Uuid::parse_str("00000000-0000-0000-0000-000000000003").unwrap();
let matches = vec![
MatchRecord {
child_id: higher_edge_key,
parent_id: child,
jaccard_similarity: 0.95,
size_difference: 99,
size_difference_pct: 0.99,
},
MatchRecord {
child_id: child,
parent_id: root,
jaccard_similarity: 0.95,
size_difference: 7,
size_difference_pct: 0.07,
},
];
let (resolved, _, _) = resolve_transitivity(&matches);
let child_assignment = resolved
.iter()
.find(|m| m.child_id == child)
.expect("child should resolve to the root");
assert_eq!(child_assignment.parent_id, root);
assert_eq!(child_assignment.size_difference, 7);
assert!((child_assignment.size_difference_pct - 0.07).abs() < 0.001);
}
#[test]
fn test_is_boilerplate_empty() {
let no_patterns: Vec<String> = vec![];
assert!(is_boilerplate("", &no_patterns));
assert!(is_boilerplate(" ", &no_patterns));
assert!(is_boilerplate("\n\n\n", &no_patterns));
}
#[test]
fn test_is_boilerplate_with_patterns() {
let patterns = vec".to_string(),
"/lw)".to_string(),
"Powered by Discuz!".to_string(),
];
assert!(is_boilerplate("Hello {{name}} world", &patterns));
assert!(is_boilerplate("${variable} test", &patterns));
assert!(is_boilerplate("Some text with }} in it", &patterns));
assert!(is_boilerplate("[EN](/goEnSite) some content", &patterns));
assert!(is_boilerplate(
"* [论文助手](/lw) * [文献直达](/zd)",
&patterns
));
assert!(is_boilerplate(
"|小黑屋|枞阳县人民医院 Powered by Discuz!",
&patterns
));
}
#[test]
fn test_is_boilerplate_navigation_heuristic() {
let no_patterns: Vec<String> = vec![];
let nav_content = (0..30)
.map(|i| format!("Link{}", i))
.collect::<Vec<_>>()
.join("\n");
assert!(is_boilerplate(&nav_content, &no_patterns));
}
#[test]
fn test_is_boilerplate_normal_content() {
let no_patterns: Vec<String> = vec![];
let article = "This is a normal article about something interesting. \
It contains multiple sentences and has meaningful content. \
The paragraphs are reasonably long and provide value to readers.";
assert!(!is_boilerplate(article, &no_patterns));
}
#[test]
fn test_is_boilerplate_pattern_with_newline() {
let patterns = vec!["首页\n新闻".to_string()];
assert!(is_boilerplate("首页\n新闻金昌要闻", &patterns));
}
}