use rand::SeedableRng;
use rand::rngs::StdRng;
use rand::seq::SliceRandom;
use rayon::prelude::*;
use crate::prelude::*;
use super::collect::collect_metacell_profiles;
use super::direct::{DirectMetacellsResult, compute_direct_metacells};
use super::params::{DacParams, MetacellsParams, RareParams};
use super::pile::Pile;
use super::rare::{RareGeneModules, find_rare_gene_modules};
const UNBOUNDED_LEVEL: usize = 1000;
#[derive(Debug, Clone)]
pub struct DacMetacellsResult {
pub metacell_of_cell: Vec<i32>,
pub dissolved_of_cell: Vec<bool>,
pub level_of_cell: Vec<i32>,
pub n_metacells: usize,
pub selected_genes: Vec<usize>,
pub rare_module_of_cell: Vec<i32>,
pub rare_module_of_gene: Vec<i32>,
}
enum Source<'a, S> {
Cells {
reader: &'a S,
cell_indices: &'a [usize],
mass: &'a [f32],
n_genes: usize,
},
Groups {
counts: &'a CompressedSparseData2<u32, f32>,
mass: &'a [f32],
},
}
impl<S: SingleCellReading> Source<'_, S> {
fn mass(&self) -> &[f32] {
match self {
Source::Cells { mass, .. } | Source::Groups { mass, .. } => mass,
}
}
fn n_genes(&self) -> usize {
match self {
Source::Cells { n_genes, .. } => *n_genes,
Source::Groups { counts, .. } => counts.shape.1,
}
}
fn rows(&self, elements: &[usize]) -> Result<CompressedSparseData2<u32, f32>, ScErrors> {
match self {
Source::Cells {
reader,
cell_indices,
..
} => {
let globals: Vec<usize> = elements.iter().map(|&e| cell_indices[e]).collect();
Ok(reader.read_cells_sparse(&globals, &DataLayerReturn::Raw)?)
}
Source::Groups { counts, .. } => Ok(counts.slice_rows(elements)?),
}
}
fn pile(&self, elements: &[usize]) -> Result<Pile, ScErrors> {
let mass = elements.iter().map(|&e| self.mass()[e]).collect();
Pile::from_raw(self.rows(elements)?, elements.to_vec(), Some(mass))
}
}
struct State {
metacell: Vec<i32>,
dissolved: Vec<bool>,
level: Vec<i32>,
selected: Vec<bool>,
}
impl State {
fn new(n: usize, n_genes: usize) -> Self {
Self {
metacell: vec![-1; n],
dissolved: vec![false; n],
level: vec![-1; n],
selected: vec![false; n_genes],
}
}
}
#[derive(Clone)]
struct Ctx {
direct: MetacellsParams,
dac: DacParams,
target_pile_size: usize,
seed: usize,
verbose: bool,
}
impl Ctx {
fn must_cover(&self) -> Self {
let mut ctx = self.clone();
ctx.direct.must_complete_cover = true;
ctx
}
}
fn round_count(x: f64) -> usize {
if x.is_finite() && x > 0.0 {
x.round_ties_even() as usize
} else {
0
}
}
pub fn compute_target_pile_size(
mass: &[f32],
target_metacell_size: usize,
target_metacell_umis: f64,
dac: &DacParams,
) -> usize {
let by_cells = target_metacell_size * dac.target_metacells_in_pile;
let mean = mass.iter().map(|&m| m as f64).sum::<f64>() / mass.len().max(1) as f64;
let by_umis = round_count(target_metacell_umis / mean * dac.target_metacells_in_pile as f64);
by_cells
.max(by_umis)
.max(dac.min_target_pile_size)
.min(dac.max_target_pile_size)
}
pub fn random_piles(n: usize, target_pile_size: usize, seed: u64) -> Vec<usize> {
let ratio = n as f64 / target_pile_size as f64;
let few = (ratio.floor() as usize).max(1);
let many = ratio.ceil() as usize;
let piles_count = if few == many {
few
} else {
let few_factor = (n as f64 / few as f64) / target_pile_size as f64;
let many_factor = target_pile_size as f64 / (n as f64 / many as f64);
if few_factor < many_factor { few } else { many }
};
let minimal = n / piles_count;
let extra = n - minimal * piles_count;
let mut pile_of: Vec<usize> = (0..extra)
.chain((0..piles_count).flat_map(|p| std::iter::repeat_n(p, minimal)))
.collect();
pile_of.shuffle(&mut StdRng::seed_from_u64(seed));
pile_of
}
fn bucket(elements: &[usize], pile_of: &[usize]) -> Vec<Vec<usize>> {
let n_piles = pile_of.iter().copied().max().map_or(0, |m| m + 1);
let mut piles = vec![Vec::new(); n_piles];
for (&e, &p) in elements.iter().zip(pile_of) {
piles[p].push(e);
}
piles
}
fn random_subset_piles(subset: &[usize], ctx: &Ctx) -> Vec<Vec<usize>> {
bucket(
subset,
&random_piles(subset.len(), ctx.target_pile_size, ctx.seed as u64),
)
}
fn run_piles<S: SingleCellReading>(
source: &Source<S>,
state: &mut State,
piles: &[Vec<usize>],
counts: &mut usize,
level: usize,
ctx: &Ctx,
) -> Result<(), ScErrors> {
let mut direct = ctx.direct.clone();
if piles.len() > 1 {
direct.must_complete_cover = false;
}
let run = |elements: &Vec<usize>| -> Result<DirectMetacellsResult, ScErrors> {
let mut pile = source.pile(elements)?;
compute_direct_metacells(&mut pile, &direct, ctx.seed, false)
};
for chunk in piles.chunks(ctx.dac.max_parallel_piles.max(1)) {
let results: Vec<DirectMetacellsResult> =
chunk.par_iter().map(run).collect::<Result<_, _>>()?;
for (elements, result) in chunk.iter().zip(results) {
for (pos, &e) in elements.iter().enumerate() {
let m = result.metacell_of_cell[pos];
state.metacell[e] = if m >= 0 { m + *counts as i32 } else { -1 };
state.dissolved[e] = result.dissolved_of_cell[pos];
state.level[e] = level as i32;
}
for &g in &result.selected_genes {
state.selected[g] = true;
}
*counts += result.n_metacells;
}
}
if ctx.verbose {
println!(
" level {level}: {} piles of {} elements, {} metacells so far",
piles.len(),
piles.iter().map(|p| p.len()).sum::<usize>(),
*counts
);
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn run_in_levels<S: SingleCellReading>(
source: &Source<S>,
state: &mut State,
subset: &[usize],
mut piles: Vec<Vec<usize>>,
counts: &mut usize,
mut level: usize,
max_level: usize,
ctx: &Ctx,
) -> Result<(), ScErrors> {
let single_pile = piles.len() == 1;
let mut was_last = false;
let mut subset = subset.to_vec();
loop {
let is_last = if level + 1 > max_level {
true
} else if single_pile {
std::mem::replace(&mut was_last, true)
} else {
false
};
run_piles(source, state, &piles, counts, level + 1, ctx)?;
if is_last {
return Ok(());
}
level += 1;
subset.retain(|&e| state.metacell[e] < 0);
if subset.is_empty() {
return Ok(());
}
piles = random_subset_piles(&subset, ctx);
}
}
fn group_piles<S: SingleCellReading>(
source: &Source<S>,
state: &mut State,
subset: &[usize],
ctx: &Ctx,
) -> Result<Vec<Vec<usize>>, ScErrors> {
let n_metacells = subset
.iter()
.map(|&e| state.metacell[e])
.max()
.map_or(0, |m| (m + 1).max(0) as usize);
let mut members: Vec<Vec<usize>> = vec![Vec::new(); n_metacells];
for &e in subset {
let m = state.metacell[e];
debug_assert!(m >= 0, "preliminary phase must cover every element");
members[m as usize].push(e);
}
let (profiles, sizes) = collect_metacell_profiles(
|elements| source.rows(elements),
&members,
source.n_genes(),
ctx.target_pile_size,
)?;
let dac = &ctx.dac;
let mut direct = ctx.direct.clone();
direct.target_metacell_umis = ctx.target_pile_size as u64;
direct.target_metacell_size = dac.target_metacells_in_pile;
direct.knn.k_size_factor = dac.piles_knn_k_size_factor;
direct.partition.min_split_size_factor = dac.piles_min_split_size_factor;
direct.partition.max_merge_size_factor = dac.piles_max_merge_size_factor;
direct.similarity = dac.groups_similarity.clone();
direct.dissolve.min_robust_size_factor = dac.piles_min_robust_size_factor;
direct.must_complete_cover = true;
let group_ctx = Ctx {
target_pile_size: compute_target_pile_size(
&sizes,
dac.target_metacells_in_pile,
ctx.target_pile_size as f64,
dac,
),
direct,
dac: dac.clone(),
seed: ctx.seed,
verbose: ctx.verbose,
};
let group_source: Source<S> = Source::Groups {
counts: &profiles,
mass: &sizes,
};
let mut group_state = State::new(n_metacells, source.n_genes());
let all: Vec<usize> = (0..n_metacells).collect();
let mut group_counts = 0;
run_subset(
&group_source,
&mut group_state,
&all,
0,
&mut group_counts,
&group_ctx,
)?;
for (s, g) in state.selected.iter_mut().zip(&group_state.selected) {
*s |= *g;
}
let pile_of: Vec<usize> = subset
.iter()
.map(|&e| group_state.metacell[state.metacell[e] as usize] as usize)
.collect();
Ok(bucket(subset, &pile_of))
}
fn run_subset<S: SingleCellReading>(
source: &Source<S>,
state: &mut State,
subset: &[usize],
level: usize,
counts: &mut usize,
ctx: &Ctx,
) -> Result<(), ScErrors> {
if subset.is_empty() {
return Ok(());
}
for &e in subset {
state.metacell[e] = -1;
}
let preliminary = random_subset_piles(subset, ctx);
if preliminary.len() == 1 {
return run_in_levels(
source,
state,
subset,
preliminary,
counts,
level,
UNBOUNDED_LEVEL,
ctx,
);
}
let cover = ctx.must_cover();
let mut preliminary_counts = 0;
run_in_levels(
source,
state,
subset,
preliminary,
&mut preliminary_counts,
level,
UNBOUNDED_LEVEL,
&cover,
)?;
let final_piles = group_piles(source, state, subset, &cover)?;
if ctx.dac.quick_and_dirty {
let max_level = if ctx.direct.must_complete_cover {
UNBOUNDED_LEVEL
} else {
1
};
return run_in_levels(
source,
state,
subset,
final_piles,
counts,
level,
max_level,
ctx,
);
}
run_in_levels(
source,
state,
subset,
final_piles,
counts,
level,
level,
&cover,
)?;
if level == 0 || ctx.direct.must_complete_cover {
let outliers: Vec<usize> = subset
.iter()
.copied()
.filter(|&e| state.metacell[e] < 0)
.collect();
run_subset(source, state, &outliers, level + 1, counts, ctx)?;
}
Ok(())
}
pub fn compute_dac_metacells<S: SingleCellReading>(
reader: &S,
cell_indices: &[usize],
mass_per_cell: Option<&[f32]>,
params: &MetacellsParams,
dac: &DacParams,
seed: usize,
verbose: bool,
) -> Result<DacMetacellsResult, ScErrors> {
let n = cell_indices.len();
if n == 0 {
return Err(ScErrors::Mc2EmptyPile);
}
let mass: Vec<f32> = match mass_per_cell {
Some(m) if m.len() != n => {
return Err(ScErrors::Mc2LengthMismatch {
what: "mass_per_cell",
expected: n,
got: m.len(),
});
}
Some(m) => m.to_vec(),
None => reader
.read_cell_library_sizes(cell_indices)?
.into_iter()
.map(|l| l as f32)
.collect(),
};
let n_genes = reader.get_header().total_genes;
let target_pile_size = dac.target_pile_size.unwrap_or_else(|| {
compute_target_pile_size(
&mass,
params.target_metacell_size,
params.target_metacell_umis as f64,
dac,
)
});
if verbose {
println!("Divide and conquer over {n} cells, target pile size {target_pile_size}");
}
let source = Source::Cells {
reader,
cell_indices,
mass: &mass,
n_genes,
};
let ctx = Ctx {
direct: params.clone(),
dac: dac.clone(),
target_pile_size,
seed,
verbose,
};
let mut state = State::new(n, n_genes);
let mut counts = 0;
let mut rare = RareGeneModules {
module_of_cell: vec![-1; n],
module_of_gene: vec![-1; n_genes],
rare_gene_mask: vec![false; n_genes],
n_modules: 0,
};
if let Some(rare_params) = &dac.rare {
rare = find_rare_gene_modules(
|elements| source.rows(elements),
n,
n_genes,
params.select.lateral_gene_mask.as_deref(),
params.target_metacell_size,
target_pile_size,
rare_params,
verbose,
)?;
if rare.n_modules > 0 {
run_rare_piles(&source, &mut state, &rare, &mut counts, rare_params, &ctx)?;
}
}
let common: Vec<usize> = (0..n).filter(|&e| state.metacell[e] < 0).collect();
run_subset(&source, &mut state, &common, 0, &mut counts, &ctx)?;
Ok(DacMetacellsResult {
metacell_of_cell: state.metacell,
dissolved_of_cell: state.dissolved,
level_of_cell: state.level,
n_metacells: counts,
selected_genes: (0..n_genes)
.filter(|&g| state.selected[g] || rare.rare_gene_mask[g])
.collect(),
rare_module_of_cell: rare.module_of_cell,
rare_module_of_gene: rare.module_of_gene,
})
}
fn run_rare_piles<S: SingleCellReading>(
source: &Source<S>,
state: &mut State,
rare: &RareGeneModules,
counts: &mut usize,
rare_params: &RareParams,
ctx: &Ctx,
) -> Result<(), ScErrors> {
let mut piles = vec![Vec::new(); rare.n_modules];
for (e, &m) in rare.module_of_cell.iter().enumerate() {
if m >= 0 {
piles[m as usize].push(e);
}
}
let mut rare_ctx = ctx.clone();
rare_ctx.direct.deviants.max_cell_fraction = rare_params.deviants_max_cell_fraction;
rare_ctx.direct.dissolve.min_robust_size_factor = rare_params.dissolve_min_robust_size_factor;
rare_ctx.direct.dissolve.min_convincing_gene_fold_factor =
rare_params.dissolve_min_convincing_gene_fold_factor;
rare_ctx.direct.must_complete_cover = false;
run_piles(source, state, &piles, counts, 0, &rare_ctx)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_random_piles_sizes_and_reproducibility() {
let piles = random_piles(25, 10, 7);
let mut sizes = [0usize; 3];
for &p in &piles {
sizes[p] += 1;
}
assert_eq!(sizes, [9, 8, 8]);
assert_eq!(piles, random_piles(25, 10, 7));
}
#[test]
fn test_random_piles_small_input_is_one_pile() {
assert!(random_piles(5, 100, 1).iter().all(|&p| p == 0));
}
#[test]
fn test_target_pile_size_clamps() {
let dac = DacParams::default();
let mass = vec![2000.0_f32; 10];
assert_eq!(compute_target_pile_size(&mass, 48, 160_000.0, &dac), 8000);
let mass = vec![100.0_f32; 10];
assert_eq!(compute_target_pile_size(&mass, 48, 160_000.0, &dac), 24000);
}
}