use rayon::prelude::*;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use super::super::{
init::domain::{Domain, SpatialBoundType},
solver::PoissonSolver,
types::*,
};
pub struct Multigrid {
pub levels: usize,
pub shape: [usize; 3],
pub dx: [f64; 3],
pub bc: SpatialBoundType,
pub n_smooth: usize,
pub tolerance: f64,
progress: Option<Arc<super::super::progress::StepProgress>>,
}
impl Multigrid {
pub fn new(domain: &Domain, levels: usize, smoothing_steps: usize) -> Self {
let shape = [
domain.spatial_res.x1 as usize,
domain.spatial_res.x2 as usize,
domain.spatial_res.x3 as usize,
];
let dx = domain.dx();
let bc = domain.spatial_bc.clone();
let min_dim = shape[0].min(shape[1]).min(shape[2]);
let max_levels = if min_dim >= 2 {
let mut max_l = 1usize;
let mut s = min_dim;
while s / 2 >= 2 {
s /= 2;
max_l += 1;
}
max_l
} else {
1
};
let levels = levels.min(max_levels);
Self {
levels,
shape,
dx,
bc,
n_smooth: smoothing_steps,
tolerance: 1e-10,
progress: None,
}
}
}
#[inline(always)]
fn idx(ix: usize, iy: usize, iz: usize, shape: [usize; 3]) -> usize {
ix * shape[1] * shape[2] + iy * shape[2] + iz
}
fn smooth_red_black(
phi: &mut [f64],
rhs: &[f64],
shape: [usize; 3],
dx: [f64; 3],
n_sweeps: usize,
bc: &SpatialBoundType,
) {
let [nx, ny, nz] = shape;
let n_total = nx * ny * nz;
let inv_dx2_x = 1.0 / (dx[0] * dx[0]);
let inv_dx2_y = 1.0 / (dx[1] * dx[1]);
let inv_dx2_z = 1.0 / (dx[2] * dx[2]);
let diag = 2.0 * inv_dx2_x + 2.0 * inv_dx2_y + 2.0 * inv_dx2_z;
let is_periodic = matches!(bc, SpatialBoundType::Periodic);
let (red_indices, black_indices): (Vec<_>, Vec<_>) = (0..n_total)
.into_par_iter()
.fold(
|| (Vec::new(), Vec::new()),
|(mut red, mut black), flat| {
let ix = flat / (ny * nz);
let iy = (flat / nz) % ny;
let iz = flat % nz;
if (ix + iy + iz) % 2 == 0 {
red.push((flat, ix, iy, iz));
} else {
black.push((flat, ix, iy, iz));
}
(red, black)
},
)
.reduce(
|| (Vec::new(), Vec::new()),
|(mut r1, mut b1), (r2, b2)| {
r1.extend(r2);
b1.extend(b2);
(r1, b1)
},
);
let parity_sets = [&red_indices, &black_indices];
for _sweep in 0..n_sweeps {
for indices in &parity_sets {
let new_values: Vec<(usize, f64)> = indices
.par_iter()
.map(|&(flat, ix, iy, iz)| {
let (phi_xm, phi_xp, phi_ym, phi_yp, phi_zm, phi_zp);
if is_periodic {
let ixm = if ix == 0 { nx - 1 } else { ix - 1 };
let ixp = if ix == nx - 1 { 0 } else { ix + 1 };
let iym = if iy == 0 { ny - 1 } else { iy - 1 };
let iyp = if iy == ny - 1 { 0 } else { iy + 1 };
let izm = if iz == 0 { nz - 1 } else { iz - 1 };
let izp = if iz == nz - 1 { 0 } else { iz + 1 };
phi_xm = phi[idx(ixm, iy, iz, shape)];
phi_xp = phi[idx(ixp, iy, iz, shape)];
phi_ym = phi[idx(ix, iym, iz, shape)];
phi_yp = phi[idx(ix, iyp, iz, shape)];
phi_zm = phi[idx(ix, iy, izm, shape)];
phi_zp = phi[idx(ix, iy, izp, shape)];
} else {
phi_xm = if ix > 0 {
phi[idx(ix - 1, iy, iz, shape)]
} else {
0.0
};
phi_xp = if ix < nx - 1 {
phi[idx(ix + 1, iy, iz, shape)]
} else {
0.0
};
phi_ym = if iy > 0 {
phi[idx(ix, iy - 1, iz, shape)]
} else {
0.0
};
phi_yp = if iy < ny - 1 {
phi[idx(ix, iy + 1, iz, shape)]
} else {
0.0
};
phi_zm = if iz > 0 {
phi[idx(ix, iy, iz - 1, shape)]
} else {
0.0
};
phi_zp = if iz < nz - 1 {
phi[idx(ix, iy, iz + 1, shape)]
} else {
0.0
};
}
let sum_neighbors = (phi_xm + phi_xp) * inv_dx2_x
+ (phi_ym + phi_yp) * inv_dx2_y
+ (phi_zm + phi_zp) * inv_dx2_z;
(flat, (sum_neighbors - rhs[flat]) / diag)
})
.collect();
for (flat, val) in new_values {
phi[flat] = val;
}
}
}
}
fn residual(
phi: &[f64],
rhs: &[f64],
shape: [usize; 3],
dx: [f64; 3],
bc: &SpatialBoundType,
) -> Vec<f64> {
let [nx, ny, nz] = shape;
let n_total = nx * ny * nz;
let inv_dx2_x = 1.0 / (dx[0] * dx[0]);
let inv_dx2_y = 1.0 / (dx[1] * dx[1]);
let inv_dx2_z = 1.0 / (dx[2] * dx[2]);
let is_periodic = matches!(bc, SpatialBoundType::Periodic);
let res: Vec<f64> = (0..n_total)
.into_par_iter()
.map(|i| {
let ix = i / (ny * nz);
let iy = (i / nz) % ny;
let iz = i % nz;
let (phi_xm, phi_xp, phi_ym, phi_yp, phi_zm, phi_zp);
if is_periodic {
let ixm = if ix == 0 { nx - 1 } else { ix - 1 };
let ixp = if ix == nx - 1 { 0 } else { ix + 1 };
let iym = if iy == 0 { ny - 1 } else { iy - 1 };
let iyp = if iy == ny - 1 { 0 } else { iy + 1 };
let izm = if iz == 0 { nz - 1 } else { iz - 1 };
let izp = if iz == nz - 1 { 0 } else { iz + 1 };
phi_xm = phi[idx(ixm, iy, iz, shape)];
phi_xp = phi[idx(ixp, iy, iz, shape)];
phi_ym = phi[idx(ix, iym, iz, shape)];
phi_yp = phi[idx(ix, iyp, iz, shape)];
phi_zm = phi[idx(ix, iy, izm, shape)];
phi_zp = phi[idx(ix, iy, izp, shape)];
} else {
phi_xm = if ix > 0 {
phi[idx(ix - 1, iy, iz, shape)]
} else {
0.0
};
phi_xp = if ix < nx - 1 {
phi[idx(ix + 1, iy, iz, shape)]
} else {
0.0
};
phi_ym = if iy > 0 {
phi[idx(ix, iy - 1, iz, shape)]
} else {
0.0
};
phi_yp = if iy < ny - 1 {
phi[idx(ix, iy + 1, iz, shape)]
} else {
0.0
};
phi_zm = if iz > 0 {
phi[idx(ix, iy, iz - 1, shape)]
} else {
0.0
};
phi_zp = if iz < nz - 1 {
phi[idx(ix, iy, iz + 1, shape)]
} else {
0.0
};
}
let laplacian = (phi_xm + phi_xp - 2.0 * phi[i]) * inv_dx2_x
+ (phi_ym + phi_yp - 2.0 * phi[i]) * inv_dx2_y
+ (phi_zm + phi_zp - 2.0 * phi[i]) * inv_dx2_z;
rhs[i] - laplacian
})
.collect();
res
}
fn restrict(fine: &[f64], fine_shape: [usize; 3]) -> (Vec<f64>, [usize; 3]) {
let [fnx, fny, fnz] = fine_shape;
let coarse_shape = [fnx / 2, fny / 2, fnz / 2];
let [cnx, cny, cnz] = coarse_shape;
let coarse: Vec<f64> = (0..cnx * cny * cnz)
.into_par_iter()
.map(|flat| {
let ci = flat / (cny * cnz);
let cj = (flat / cnz) % cny;
let ck = flat % cnz;
let fi = 2 * ci;
let fj = 2 * cj;
let fk = 2 * ck;
let mut val = 0.0;
for di in 0..3i32 {
for dj in 0..3i32 {
for dk in 0..3i32 {
let si = fi as i32 + di - 1;
let sj = fj as i32 + dj - 1;
let sk = fk as i32 + dk - 1;
if si < 0
|| si >= fnx as i32
|| sj < 0
|| sj >= fny as i32
|| sk < 0
|| sk >= fnz as i32
{
continue;
}
let si = si as usize;
let sj = sj as usize;
let sk = sk as usize;
let dist = (di != 1) as u32 + (dj != 1) as u32 + (dk != 1) as u32;
let weight = 1.0 / (1u32 << (3 + dist)) as f64;
val += weight * fine[idx(si, sj, sk, fine_shape)];
}
}
}
val
})
.collect();
(coarse, coarse_shape)
}
fn prolongate(coarse: &[f64], coarse_shape: [usize; 3], fine_shape: [usize; 3]) -> Vec<f64> {
let [cnx, cny, cnz] = coarse_shape;
let [fnx, fny, fnz] = fine_shape;
let n_fine = fnx * fny * fnz;
let fine: Vec<f64> = (0..n_fine)
.into_par_iter()
.map(|flat| {
let fi = flat / (fny * fnz);
let fj = (flat / fnz) % fny;
let fk = flat % fnz;
let (ci0, wx) = coarse_interp_index(fi, cnx);
let (cj0, wy) = coarse_interp_index(fj, cny);
let (ck0, wz) = coarse_interp_index(fk, cnz);
let ci1 = (ci0 + 1).min(cnx - 1);
let cj1 = (cj0 + 1).min(cny - 1);
let ck1 = (ck0 + 1).min(cnz - 1);
let c000 = coarse[idx(ci0, cj0, ck0, coarse_shape)];
let c100 = coarse[idx(ci1, cj0, ck0, coarse_shape)];
let c010 = coarse[idx(ci0, cj1, ck0, coarse_shape)];
let c110 = coarse[idx(ci1, cj1, ck0, coarse_shape)];
let c001 = coarse[idx(ci0, cj0, ck1, coarse_shape)];
let c101 = coarse[idx(ci1, cj0, ck1, coarse_shape)];
let c011 = coarse[idx(ci0, cj1, ck1, coarse_shape)];
let c111 = coarse[idx(ci1, cj1, ck1, coarse_shape)];
c000 * (1.0 - wx) * (1.0 - wy) * (1.0 - wz)
+ c100 * wx * (1.0 - wy) * (1.0 - wz)
+ c010 * (1.0 - wx) * wy * (1.0 - wz)
+ c110 * wx * wy * (1.0 - wz)
+ c001 * (1.0 - wx) * (1.0 - wy) * wz
+ c101 * wx * (1.0 - wy) * wz
+ c011 * (1.0 - wx) * wy * wz
+ c111 * wx * wy * wz
})
.collect();
fine
}
#[inline]
fn coarse_interp_index(fi: usize, cn: usize) -> (usize, f64) {
let ci0 = fi / 2;
let ci0 = ci0.min(cn.saturating_sub(1));
let w = (fi as f64 / 2.0) - ci0 as f64;
(ci0, w)
}
#[allow(clippy::too_many_arguments)]
fn v_cycle(
phi: &mut [f64],
rhs: &[f64],
shape: [usize; 3],
dx: [f64; 3],
level: usize,
max_level: usize,
n_pre: usize,
n_post: usize,
bc: &SpatialBoundType,
) {
smooth_red_black(phi, rhs, shape, dx, n_pre, bc);
if level >= max_level - 1 {
smooth_red_black(phi, rhs, shape, dx, 50, bc);
return;
}
let res = residual(phi, rhs, shape, dx, bc);
let (rhs_coarse, coarse_shape) = restrict(&res, shape);
let coarse_dx = [dx[0] * 2.0, dx[1] * 2.0, dx[2] * 2.0];
let mut e_coarse = vec![0.0; coarse_shape[0] * coarse_shape[1] * coarse_shape[2]];
v_cycle(
&mut e_coarse,
&rhs_coarse,
coarse_shape,
coarse_dx,
level + 1,
max_level,
n_pre,
n_post,
bc,
);
let correction = prolongate(&e_coarse, coarse_shape, shape);
phi.par_iter_mut()
.zip(correction.par_iter())
.for_each(|(p, &c)| *p += c);
smooth_red_black(phi, rhs, shape, dx, n_post, bc);
}
#[inline]
fn l2_norm(v: &[f64]) -> f64 {
v.iter().map(|x| x * x).sum::<f64>().sqrt()
}
impl PoissonSolver for Multigrid {
fn set_progress(&mut self, p: std::sync::Arc<super::super::progress::StepProgress>) {
self.progress = Some(p);
}
fn solve(&self, density: &DensityField, g: f64) -> PotentialField {
let _span = tracing::info_span!("multigrid_solve").entered();
let [nx, ny, nz] = self.shape;
let n_total = nx * ny * nz;
let four_pi_g = 4.0 * std::f64::consts::PI * g;
let rhs: Vec<f64> = density.data.iter().map(|&r| four_pi_g * r).collect();
let rhs_norm = l2_norm(&rhs);
let mut phi = vec![0.0; n_total];
let max_iter = 100u64;
for _iter in 0..max_iter {
if let Some(ref p) = self.progress {
p.set_intra_progress(_iter, max_iter);
}
v_cycle(
&mut phi,
&rhs,
self.shape,
self.dx,
0,
self.levels,
self.n_smooth,
self.n_smooth,
&self.bc,
);
let res = residual(&phi, &rhs, self.shape, self.dx, &self.bc);
let res_norm = l2_norm(&res);
if rhs_norm < 1e-15 {
break;
}
if res_norm / rhs_norm < self.tolerance {
break;
}
}
if matches!(self.bc, SpatialBoundType::Periodic) {
let mean = phi.iter().sum::<f64>() / n_total as f64;
for p in phi.iter_mut() {
*p -= mean;
}
}
PotentialField {
data: phi,
shape: self.shape,
}
}
fn compute_acceleration(&self, potential: &PotentialField) -> AccelerationField {
super::utils::finite_difference_acceleration(potential, &self.dx)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::tooling::core::init::domain::{Domain, SpatialBoundType, VelocityBoundType};
use std::f64::consts::PI;
fn periodic_domain(n: i128) -> Domain {
Domain::builder()
.spatial_extent(1.0)
.velocity_extent(1.0)
.spatial_resolution(n)
.velocity_resolution(4)
.t_final(1.0)
.spatial_bc(SpatialBoundType::Periodic)
.velocity_bc(VelocityBoundType::Open)
.build()
.unwrap()
}
#[test]
fn multigrid_sin_solution() {
let domain = periodic_domain(32);
let mg = Multigrid::new(&domain, 4, 3);
let [nx, ny, nz] = mg.shape;
let dx = mg.dx;
let mut rho = vec![0.0; nx * ny * nz];
for ix in 0..nx {
for iy in 0..ny {
for iz in 0..nz {
let x = -1.0 + (ix as f64 + 0.5) * dx[0];
let y = -1.0 + (iy as f64 + 0.5) * dx[1];
let z = -1.0 + (iz as f64 + 0.5) * dx[2];
let i = ix * ny * nz + iy * nz + iz;
rho[i] = -3.0 * PI * PI * (PI * x).sin() * (PI * y).sin() * (PI * z).sin()
/ (4.0 * PI);
}
}
}
let density = DensityField {
data: rho,
shape: [nx, ny, nz],
};
let pot = mg.solve(&density, 1.0);
let mid = nx / 2;
let i = mid * ny * nz + mid * nz + mid;
let x = -1.0 + (mid as f64 + 0.5) * dx[0];
let expected = (PI * x).sin().powi(3); let err = (pot.data[i] - expected).abs();
assert!(
err < 0.5,
"Multigrid sin solution error {err} at center (expected {expected}, got {})",
pot.data[i]
);
}
#[test]
fn multigrid_convergence_order() {
let domain8 = periodic_domain(8);
let domain16 = periodic_domain(16);
let mg8 = Multigrid::new(&domain8, 3, 3);
let mg16 = Multigrid::new(&domain16, 4, 3);
let make_rho = |shape: [usize; 3], dx: [f64; 3]| -> Vec<f64> {
let mut rho = vec![0.0; shape[0] * shape[1] * shape[2]];
for ix in 0..shape[0] {
for iy in 0..shape[1] {
for iz in 0..shape[2] {
let x = -1.0 + (ix as f64 + 0.5) * dx[0];
let y = -1.0 + (iy as f64 + 0.5) * dx[1];
let z = -1.0 + (iz as f64 + 0.5) * dx[2];
rho[ix * shape[1] * shape[2] + iy * shape[2] + iz] =
-3.0 * PI * PI * (PI * x).sin() * (PI * y).sin() * (PI * z).sin()
/ (4.0 * PI);
}
}
}
rho
};
let rho8 = make_rho(mg8.shape, mg8.dx);
let rho16 = make_rho(mg16.shape, mg16.dx);
let pot8 = mg8.solve(
&DensityField {
data: rho8,
shape: mg8.shape,
},
1.0,
);
let pot16 = mg16.solve(
&DensityField {
data: rho16,
shape: mg16.shape,
},
1.0,
);
let err = |pot: &PotentialField, dx: [f64; 3]| -> f64 {
let [nx, ny, nz] = pot.shape;
let mut max_e = 0.0f64;
for ix in 0..nx {
for iy in 0..ny {
for iz in 0..nz {
let x = -1.0 + (ix as f64 + 0.5) * dx[0];
let y = -1.0 + (iy as f64 + 0.5) * dx[1];
let z = -1.0 + (iz as f64 + 0.5) * dx[2];
let exact = (PI * x).sin() * (PI * y).sin() * (PI * z).sin();
let e = (pot.data[ix * ny * nz + iy * nz + iz] - exact).abs();
max_e = max_e.max(e);
}
}
}
max_e
};
let e8 = err(&pot8, mg8.dx);
let e16 = err(&pot16, mg16.dx);
if e8 > 1e-10 && e16 > 1e-10 {
let ratio = e8 / e16;
assert!(
ratio > 1.5,
"Convergence ratio {ratio} too low (e8={e8}, e16={e16})"
);
}
}
#[test]
fn multigrid_vs_fft() {
use crate::tooling::core::poisson::fft::FftPoisson;
let domain = periodic_domain(16);
let mg = Multigrid::new(&domain, 4, 3);
let fft = FftPoisson::new(&domain);
let [nx, ny, nz] = mg.shape;
let dx = mg.dx;
let mut rho = vec![0.0; nx * ny * nz];
for ix in 0..nx {
for iy in 0..ny {
for iz in 0..nz {
let x = -1.0 + (ix as f64 + 0.5) * dx[0];
let y = -1.0 + (iy as f64 + 0.5) * dx[1];
let z = -1.0 + (iz as f64 + 0.5) * dx[2];
rho[ix * ny * nz + iy * nz + iz] =
(PI * x).sin() * (PI * y).sin() * (PI * z).sin();
}
}
}
let density = DensityField {
data: rho,
shape: [nx, ny, nz],
};
let pot_mg = mg.solve(&density, 1.0);
let pot_fft = fft.solve(&density, 1.0);
let mean_mg: f64 = pot_mg.data.iter().sum::<f64>() / pot_mg.data.len() as f64;
let mean_fft: f64 = pot_fft.data.iter().sum::<f64>() / pot_fft.data.len() as f64;
let mut max_diff = 0.0f64;
for i in 0..pot_mg.data.len() {
let diff = ((pot_mg.data[i] - mean_mg) - (pot_fft.data[i] - mean_fft)).abs();
max_diff = max_diff.max(diff);
}
let scale = pot_fft
.data
.iter()
.map(|x| x.abs())
.fold(0.0f64, f64::max)
.max(1.0);
assert!(
max_diff / scale < 0.1,
"Multigrid vs FFT max diff {max_diff} (scale {scale})"
);
}
}