Struct Corpus 

pub struct Corpus { /* private fields */ }

A corpus of L2-normalised sparse non-negative vectors in CSR form, with dimensions relabelled to a global rank (decreasing max weight). Built once; joinable at any threshold.

Implementations

impl Corpus

pub fn len(&self) -> usize

Number of vectors.

pub fn is_empty(&self) -> bool

True if the corpus has no vectors.

pub fn from_token_docs<S: AsRef<str>>(docs: &[Vec<S>]) -> Corpus

Build a corpus from token documents — each document a list of string tokens — as TF-IDF sparse vectors: dim = a distinct token, weight = (token count in doc) × ln(n / df_token) (df = number of documents containing the token). This is the principled input for a Type-3 code-clone join (documents = functions, tokens = canonicalised lines) and the shape most callers actually have. A token appearing in every document gets idf = 0 (contributes nothing), as expected.

pub fn from_rows(rows: &[Vec<(u32, f64)>]) -> Corpus

Build a corpus from raw (dim, weight) rows. Duplicate dims within a row are summed; each row is L2-normalised; dims are relabelled to a global rank by decreasing max weight. Weights are expected non-negative (the prefix-filter bound requires it — IDF weights satisfy this).

Examples found in repository

examples/simjoin_bench.rs (line 83)

fn main() {
    let n: usize = arg(1, 100_000);
    let nnz: usize = arg(2, 14);
    let ndims: usize = arg(3, 20_000);
    let t: f64 = arg(4, 0.7);
    let reps: usize = arg(5, 3);

    let rows = gen(n, nnz, ndims, 0x1234_5678_9abc_def1);
    let build0 = Instant::now();
    let corpus = Corpus::from_rows(&rows);
    let build_ms = build0.elapsed().as_secs_f64() * 1000.0;

    // Strategy diagnostic (profiling builds only): posting touches / candidates / pairs. The
    // candidates-per-pair ratio decides whether to prune harder or speed the dot up.
    #[cfg(feature = "profiling")]
    if std::env::var("STATS").is_ok() {
        let (ncand, survivors, pairs) = cosine_join_counts(&corpus, t);
        eprintln!(
            "STATS n={n} t={t} | candidates={ncand} survivors(cos_full)={survivors} pairs={pairs} \
             | prune_pass={:.4} cos_full_saved={:.4} survivor_precision={:.3}",
            survivors as f64 / ncand.max(1) as f64,
            1.0 - survivors as f64 / ncand.max(1) as f64,
            pairs as f64 / survivors.max(1) as f64,
        );
    }

    let mut ms: Vec<f64> = Vec::with_capacity(reps);
    let mut npairs = 0usize;
    for _ in 0..reps {
        let t0 = Instant::now();
        let pairs = cosine_join(&corpus, t);
        ms.push(t0.elapsed().as_secs_f64() * 1000.0);
        npairs = pairs.len();
        std::hint::black_box(&pairs);
    }
    ms.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
    eprintln!(
        "n={n} nnz={nnz} ndims={ndims} t={t} | build={build_ms:.0}ms | join: min={:.1}ms median={:.1}ms | pairs={npairs}",
        ms[0],
        ms[reps / 2],
    );
}

examples/simjoin_pypi.rs (line 72)

fn main() {
    let path: String = arg(1, "perf-local/pypi-type3.simjoin.bin".to_string());
    let t: f64 = arg(2, 0.8);
    let reps: usize = arg(3, 3);

    let mut rows = load(&path);
    // Optional row subset (SJ_NSUB) to bench the small-corpus regime (e.g. find-dup-defs scale
    // ~3216 functions) instead of the full 287k bandwidth-bound regime.
    if let Ok(nsub) = std::env::var("SJ_NSUB") {
        if let Ok(k) = nsub.parse::<usize>() {
            if rows.len() > k {
                rows.truncate(k);
            }
        }
    }
    let n = rows.len();
    let nnz: usize = rows.iter().map(Vec::len).sum();

    let b0 = Instant::now();
    let corpus = Corpus::from_rows(&rows);
    let build_ms = b0.elapsed().as_secs_f64() * 1000.0;

    #[cfg(feature = "profiling")]
    if std::env::var("STATS").is_ok() {
        let (ncand, survivors, pairs) = cosine_join_counts(&corpus, t);
        eprintln!(
            "STATS n={n} t={t} | candidates={ncand} survivors(cos_full)={survivors} pairs={pairs} \
             | prune_pass={:.4} survivor_precision={:.3}",
            survivors as f64 / ncand.max(1) as f64,
            pairs as f64 / survivors.max(1) as f64,
        );
    }

    let mut ms: Vec<f64> = Vec::with_capacity(reps);
    let mut npairs = 0usize;
    for _ in 0..reps {
        let t0 = Instant::now();
        let pairs = cosine_join(&corpus, t);
        ms.push(t0.elapsed().as_secs_f64() * 1000.0);
        npairs = pairs.len();
        std::hint::black_box(&pairs);
    }
    ms.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
    eprintln!(
        "pypi-type3 n={n} nnz_total={nnz} mean_nnz={:.1} t={t} | build={build_ms:.0}ms | \
         join: min={:.1}ms median={:.1}ms | pairs={npairs}",
        nnz as f64 / n as f64,
        ms[0],
        ms[reps / 2],
    );
}