pub mod cleanup;
pub mod cli;
pub mod db;
pub mod dedupe;
pub mod disk_dedupe;
pub mod lsh;
pub mod minhash;
pub mod sources;
pub mod storage;
pub mod tokenizer;
pub mod union_find;
pub use db::{DbConfig, DbPool, Document, DupeMatch};
pub use dedupe::{
is_boilerplate, load_junk_patterns, resolve_transitivity, run_dedupe, run_dedupe_with_source,
DedupeConfig, DedupeStats, EdgeLookupMode, InMemoryDeduplicator, IndexBuilder,
};
#[deprecated(since = "0.2.4", note = "Use IndexBuilder for Phase 1 index building")]
pub type DiskIndexBuilder = IndexBuilder;
pub use cleanup::{
find_pathological_clusters, run_cleanup, run_phase_1_5, CleanupAction, CleanupStats,
PathologicalCluster,
};
pub use cli::{Args, EdgeLookupArg};
pub use disk_dedupe::{
run_disk_dedupe, run_disk_dedupe_with_options, DiskDedupeRunOptions, DiskDedupeStats,
DiskDeduplicator,
};
pub use lsh::{DiskLSH, InMemoryLSH};
pub use minhash::{
calculate_band_hash, compute_band_hashes, jaccard_from_signatures, try_compute_band_hashes,
try_jaccard_from_signatures, RMinHash, NUM_BANDS, NUM_PERM, ROWS_PER_BAND,
};
pub use sources::{
DocumentSource, FileSystemSource, PostgresSource, SourceDocument, SourceDupeMatch, SqliteSource,
};
pub use storage::{
DatasetStorage, FilteredParentStore, MatchRecord, MatchStore, StateStore, SyncProgress,
SyncStep,
};
pub use tokenizer::{
compute_signature, compute_signature_with_ngram_width, normalize_content, tokenize,
tokenize_with_ngram_width, DEFAULT_CHAR_NGRAM_WIDTH, TOKENIZER_VERSION,
};
pub use union_find::UnionFind;