use crate::error::{Result, TemporalError};
use crate::timeseries::TimeSeriesRaster;
use scirs2_core::linalg::solve_ndarray;
use scirs2_core::ndarray::{Array1, Array2, Array3};
use serde::{Deserialize, Serialize};
use std::f64::consts::PI;
use tracing::info;
pub mod harmonic;
pub mod interpolation;
pub mod savgol;
pub mod spline;
pub mod whittaker;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum GapFillMethod {
LinearInterpolation,
SplineInterpolation,
NearestNeighbor,
HarmonicRegression,
MovingAverage,
ForwardFill,
BackwardFill,
Whittaker,
SavitzkyGolay,
}
#[derive(Debug, Clone)]
pub struct GapFillResult {
pub data: Array3<f64>,
pub filled_count: Array3<usize>,
pub quality: Option<Array3<f64>>,
}
impl GapFillResult {
#[must_use]
pub fn new(data: Array3<f64>, filled_count: Array3<usize>) -> Self {
Self {
data,
filled_count,
quality: None,
}
}
#[must_use]
pub fn with_quality(mut self, quality: Array3<f64>) -> Self {
self.quality = Some(quality);
self
}
}
pub struct GapFiller;
impl GapFiller {
pub fn fill_gaps(
ts: &TimeSeriesRaster,
method: GapFillMethod,
params: Option<GapFillParams>,
) -> Result<TimeSeriesRaster> {
let mut filled = match method {
GapFillMethod::LinearInterpolation => Self::linear_interpolation(ts)?,
GapFillMethod::SplineInterpolation => Self::spline_interpolation(ts)?,
GapFillMethod::NearestNeighbor => Self::nearest_neighbor(ts)?,
GapFillMethod::HarmonicRegression => {
let period = params.map_or(12, |p| p.harmonic_period);
Self::harmonic_regression(ts, period)?
}
GapFillMethod::MovingAverage => {
let window = params.map_or(3, |p| p.window_size);
Self::moving_average(ts, window)?
}
GapFillMethod::ForwardFill => Self::forward_fill(ts)?,
GapFillMethod::BackwardFill => Self::backward_fill(ts)?,
GapFillMethod::Whittaker => {
let lambda = params.map_or(100.0, |p| p.whittaker_lambda);
let order = params.map_or(2, |p| p.whittaker_order);
Self::whittaker_smooth(ts, lambda, order)?
}
GapFillMethod::SavitzkyGolay => {
let win = params.map_or(7, |p| p.savgol_window);
let poly = params.map_or(2, |p| p.savgol_poly_order);
Self::savitzky_golay_smooth(ts, win, poly)?
}
};
if let Some(max_gap) = params.and_then(|p| p.max_gap_size) {
Self::apply_max_gap_size(ts, &mut filled, max_gap)?;
}
Ok(filled)
}
fn apply_max_gap_size(
original: &TimeSeriesRaster,
filled: &mut TimeSeriesRaster,
max_gap: usize,
) -> Result<()> {
let (height, width, n_bands) = original
.expected_shape()
.ok_or_else(|| TemporalError::insufficient_data("No shape information"))?;
for i in 0..height {
for j in 0..width {
for k in 0..n_bands {
let orig = original.extract_pixel_timeseries(i, j, k)?;
let n = orig.len();
let mut t = 0;
while t < n {
if orig[t].is_nan() {
let start = t;
while t < n && orig[t].is_nan() {
t += 1;
}
let run_len = t - start;
if run_len > max_gap {
for (idx, entry) in filled.entries_mut().values_mut().enumerate() {
if idx >= start
&& idx < t
&& let Some(data) = &mut entry.data
{
data[[i, j, k]] = f64::NAN;
}
}
}
} else {
t += 1;
}
}
}
}
}
Ok(())
}
fn linear_interpolation(ts: &TimeSeriesRaster) -> Result<TimeSeriesRaster> {
if ts.len() < 2 {
return Err(TemporalError::insufficient_data(
"Need at least 2 observations",
));
}
let (height, width, n_bands) = ts
.expected_shape()
.ok_or_else(|| TemporalError::insufficient_data("No shape information"))?;
let mut filled_ts = ts.clone();
for i in 0..height {
for j in 0..width {
for k in 0..n_bands {
let values = ts.extract_pixel_timeseries(i, j, k)?;
let filled = Self::interpolate_linear(&values);
for (t, entry) in filled_ts.entries_mut().values_mut().enumerate() {
if let Some(data) = &mut entry.data {
data[[i, j, k]] = filled[t];
}
}
}
}
}
info!("Completed linear interpolation gap filling");
Ok(filled_ts)
}
fn interpolate_linear(values: &[f64]) -> Vec<f64> {
let mut result = values.to_vec();
for i in 0..result.len() {
if result[i].is_nan() {
let mut prev_idx = None;
for j in (0..i).rev() {
if !result[j].is_nan() {
prev_idx = Some(j);
break;
}
}
let next_idx = result[(i + 1)..]
.iter()
.position(|&v| !v.is_nan())
.map(|idx| idx + i + 1);
if let (Some(prev), Some(next)) = (prev_idx, next_idx) {
let prev_val = result[prev];
let next_val = result[next];
let weight = (i - prev) as f64 / (next - prev) as f64;
result[i] = prev_val + weight * (next_val - prev_val);
}
}
}
result
}
fn spline_interpolation(ts: &TimeSeriesRaster) -> Result<TimeSeriesRaster> {
if ts.len() < 2 {
return Err(TemporalError::insufficient_data(
"Need at least 2 observations",
));
}
let (height, width, n_bands) = ts
.expected_shape()
.ok_or_else(|| TemporalError::insufficient_data("No shape information"))?;
let mut filled_ts = ts.clone();
for i in 0..height {
for j in 0..width {
for k in 0..n_bands {
let values = ts.extract_pixel_timeseries(i, j, k)?;
let filled = spline::fill_natural_cubic_spline(&values);
for (t, entry) in filled_ts.entries_mut().values_mut().enumerate() {
if let Some(data) = &mut entry.data {
data[[i, j, k]] = filled[t];
}
}
}
}
}
info!("Completed natural cubic spline gap filling");
Ok(filled_ts)
}
fn nearest_neighbor(ts: &TimeSeriesRaster) -> Result<TimeSeriesRaster> {
if ts.is_empty() {
return Err(TemporalError::insufficient_data("Empty time series"));
}
let (height, width, n_bands) = ts
.expected_shape()
.ok_or_else(|| TemporalError::insufficient_data("No shape information"))?;
let mut filled_ts = ts.clone();
for i in 0..height {
for j in 0..width {
for k in 0..n_bands {
let values = ts.extract_pixel_timeseries(i, j, k)?;
let filled = Self::fill_nearest(&values);
for (t, entry) in filled_ts.entries_mut().values_mut().enumerate() {
if let Some(data) = &mut entry.data {
data[[i, j, k]] = filled[t];
}
}
}
}
}
info!("Completed nearest neighbor gap filling");
Ok(filled_ts)
}
fn fill_nearest(values: &[f64]) -> Vec<f64> {
let mut result = values.to_vec();
for i in 0..result.len() {
if result[i].is_nan() {
let nearest_val = result
.iter()
.enumerate()
.filter(|(_, v)| !v.is_nan())
.min_by_key(|(j, _)| i.abs_diff(*j))
.map(|(_, v)| *v)
.unwrap_or(f64::NAN);
result[i] = nearest_val;
}
}
result
}
fn harmonic_regression(ts: &TimeSeriesRaster, period: usize) -> Result<TimeSeriesRaster> {
if ts.len() < period {
return Err(TemporalError::insufficient_data(format!(
"Need at least {} observations for period {}",
period, period
)));
}
let (height, width, n_bands) = ts
.expected_shape()
.ok_or_else(|| TemporalError::insufficient_data("No shape information"))?;
let mut filled_ts = ts.clone();
for i in 0..height {
for j in 0..width {
for k in 0..n_bands {
let values = ts.extract_pixel_timeseries(i, j, k)?;
let filled = Self::fit_harmonic(&values, period);
for (t, entry) in filled_ts.entries_mut().values_mut().enumerate() {
if let Some(data) = &mut entry.data {
data[[i, j, k]] = filled[t];
}
}
}
}
}
info!("Completed harmonic regression gap filling");
Ok(filled_ts)
}
fn fit_harmonic(values: &[f64], period: usize) -> Vec<f64> {
let valid_data: Vec<(usize, f64)> = values
.iter()
.enumerate()
.filter(|(_, v)| !v.is_nan())
.map(|(i, &v)| (i, v))
.collect();
if valid_data.is_empty() {
return values.to_vec();
}
let n_valid = valid_data.len();
let mean = || valid_data.iter().map(|&(_, y)| y).sum::<f64>() / n_valid as f64;
let (a_coef, b, c) = if n_valid < 3 {
(mean(), 0.0, 0.0)
} else {
let mut design = Array2::<f64>::zeros((n_valid, 3));
let mut target = Array1::<f64>::zeros(n_valid);
for (row, &(t, y)) in valid_data.iter().enumerate() {
let phase = 2.0 * PI * (t as f64) / (period as f64);
design[[row, 0]] = 1.0;
design[[row, 1]] = phase.sin();
design[[row, 2]] = phase.cos();
target[row] = y;
}
let design_t = design.t().to_owned();
let ata = design_t.dot(&design);
let aty = design_t.dot(&target);
match solve_ndarray(&ata, &aty) {
Ok(beta) => (beta[0], beta[1], beta[2]),
Err(_) => {
(mean(), 0.0, 0.0)
}
}
};
values
.iter()
.enumerate()
.map(|(t, val)| {
let phase = 2.0 * PI * (t as f64) / (period as f64);
let fitted = a_coef + b * phase.sin() + c * phase.cos();
if val.is_nan() { fitted } else { *val }
})
.collect()
}
fn moving_average(ts: &TimeSeriesRaster, window: usize) -> Result<TimeSeriesRaster> {
if ts.len() < window {
return Err(TemporalError::insufficient_data(format!(
"Need at least {} observations",
window
)));
}
let (height, width, n_bands) = ts
.expected_shape()
.ok_or_else(|| TemporalError::insufficient_data("No shape information"))?;
let mut filled_ts = ts.clone();
for i in 0..height {
for j in 0..width {
for k in 0..n_bands {
let values = ts.extract_pixel_timeseries(i, j, k)?;
let filled = Self::fill_moving_average(&values, window);
for (t, entry) in filled_ts.entries_mut().values_mut().enumerate() {
if let Some(data) = &mut entry.data {
data[[i, j, k]] = filled[t];
}
}
}
}
}
info!("Completed moving average gap filling");
Ok(filled_ts)
}
fn fill_moving_average(values: &[f64], window: usize) -> Vec<f64> {
let mut result = values.to_vec();
let half_window = window / 2;
for i in 0..result.len() {
if result[i].is_nan() {
let start = i.saturating_sub(half_window);
let end = (i + half_window + 1).min(result.len());
let valid_values: Vec<f64> = result[start..end]
.iter()
.filter(|v| !v.is_nan())
.copied()
.collect();
if !valid_values.is_empty() {
result[i] = valid_values.iter().sum::<f64>() / valid_values.len() as f64;
}
}
}
result
}
fn forward_fill(ts: &TimeSeriesRaster) -> Result<TimeSeriesRaster> {
if ts.is_empty() {
return Err(TemporalError::insufficient_data("Empty time series"));
}
let (height, width, n_bands) = ts
.expected_shape()
.ok_or_else(|| TemporalError::insufficient_data("No shape information"))?;
let mut filled_ts = ts.clone();
for i in 0..height {
for j in 0..width {
for k in 0..n_bands {
let values = ts.extract_pixel_timeseries(i, j, k)?;
let mut last_valid = f64::NAN;
let mut filled = Vec::with_capacity(values.len());
for &value in &values {
let v: f64 = value;
if !v.is_nan() && v.is_finite() {
last_valid = v;
filled.push(v);
} else {
filled.push(last_valid);
}
}
for (t, entry) in filled_ts.entries_mut().values_mut().enumerate() {
if let Some(data) = &mut entry.data {
data[[i, j, k]] = filled[t];
}
}
}
}
}
info!("Completed forward fill");
Ok(filled_ts)
}
fn backward_fill(ts: &TimeSeriesRaster) -> Result<TimeSeriesRaster> {
if ts.is_empty() {
return Err(TemporalError::insufficient_data("Empty time series"));
}
let (height, width, n_bands) = ts
.expected_shape()
.ok_or_else(|| TemporalError::insufficient_data("No shape information"))?;
let mut filled_ts = ts.clone();
for i in 0..height {
for j in 0..width {
for k in 0..n_bands {
let values = ts.extract_pixel_timeseries(i, j, k)?;
let mut filled = values.clone();
let mut next_valid = f64::NAN;
for t in (0..values.len()).rev() {
if !values[t].is_nan() {
next_valid = values[t];
} else {
filled[t] = next_valid;
}
}
for (t, entry) in filled_ts.entries_mut().values_mut().enumerate() {
if let Some(data) = &mut entry.data {
data[[i, j, k]] = filled[t];
}
}
}
}
}
info!("Completed backward fill");
Ok(filled_ts)
}
fn whittaker_smooth(
ts: &TimeSeriesRaster,
lambda: f64,
order: usize,
) -> Result<TimeSeriesRaster> {
if ts.is_empty() {
return Err(TemporalError::insufficient_data("Empty time series"));
}
let (height, width, n_bands) = ts
.expected_shape()
.ok_or_else(|| TemporalError::insufficient_data("No shape information"))?;
let mut filled_ts = ts.clone();
for i in 0..height {
for j in 0..width {
for k in 0..n_bands {
let values = ts.extract_pixel_timeseries(i, j, k)?;
let smoothed = whittaker::smooth_whittaker(&values, lambda, order);
for (t, entry) in filled_ts.entries_mut().values_mut().enumerate() {
if let Some(data) = &mut entry.data {
data[[i, j, k]] = smoothed[t];
}
}
}
}
}
info!("Completed Whittaker smoother gap filling (lambda={lambda}, order={order})");
Ok(filled_ts)
}
fn savitzky_golay_smooth(
ts: &TimeSeriesRaster,
window: usize,
poly_order: usize,
) -> Result<TimeSeriesRaster> {
if ts.is_empty() {
return Err(TemporalError::insufficient_data("Empty time series"));
}
let (height, width, n_bands) = ts
.expected_shape()
.ok_or_else(|| TemporalError::insufficient_data("No shape information"))?;
let mut filled_ts = ts.clone();
for i in 0..height {
for j in 0..width {
for k in 0..n_bands {
let values = ts.extract_pixel_timeseries(i, j, k)?;
let smoothed = savgol::smooth_savgol(&values, window, poly_order);
for (t, entry) in filled_ts.entries_mut().values_mut().enumerate() {
if let Some(data) = &mut entry.data {
data[[i, j, k]] = smoothed[t];
}
}
}
}
}
info!("Completed Savitzky-Golay smoothing (window={window}, poly_order={poly_order})");
Ok(filled_ts)
}
}
#[derive(Debug, Clone, Copy)]
pub struct GapFillParams {
pub window_size: usize,
pub harmonic_period: usize,
pub max_gap_size: Option<usize>,
pub whittaker_lambda: f64,
pub whittaker_order: usize,
pub savgol_window: usize,
pub savgol_poly_order: usize,
}
impl Default for GapFillParams {
fn default() -> Self {
Self {
window_size: 3,
harmonic_period: 12,
max_gap_size: None,
whittaker_lambda: 100.0,
whittaker_order: 2,
savgol_window: 7,
savgol_poly_order: 2,
}
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
mod tests {
use super::*;
use crate::timeseries::TemporalMetadata;
use chrono::{DateTime, NaiveDate, Utc};
use scirs2_core::ndarray::Array3;
fn meta(day: u32) -> TemporalMetadata {
let date = NaiveDate::from_ymd_opt(2024, 1, day).expect("valid date");
let ndt = date.and_hms_opt(0, 0, 0).expect("valid time");
let ts = DateTime::from_naive_utc_and_offset(ndt, Utc);
TemporalMetadata::new(ts, date)
}
fn ts_from(values: &[f64]) -> TimeSeriesRaster {
let mut ts = TimeSeriesRaster::new();
for (idx, &v) in values.iter().enumerate() {
let raster = Array3::from_elem((1, 1, 1), v);
ts.add_raster(meta(idx as u32 + 1), raster).unwrap();
}
ts
}
fn pixel_series(ts: &TimeSeriesRaster) -> Vec<f64> {
ts.extract_pixel_timeseries(0, 0, 0).unwrap()
}
#[test]
fn test_max_gap_size_leaves_long_gaps_unfilled() {
let values = vec![
1.0,
f64::NAN, 3.0,
f64::NAN,
f64::NAN,
f64::NAN, 7.0,
];
let ts = ts_from(&values);
let params = GapFillParams {
max_gap_size: Some(1),
..GapFillParams::default()
};
let filled =
GapFiller::fill_gaps(&ts, GapFillMethod::LinearInterpolation, Some(params)).unwrap();
let out = pixel_series(&filled);
assert!(
(out[1] - 2.0).abs() < 1e-9,
"short gap should be filled, got {}",
out[1]
);
assert!(out[3].is_nan(), "long gap position 3 must stay NaN");
assert!(out[4].is_nan(), "long gap position 4 must stay NaN");
assert!(out[5].is_nan(), "long gap position 5 must stay NaN");
assert_eq!(out[0], 1.0);
assert_eq!(out[2], 3.0);
assert_eq!(out[6], 7.0);
}
#[test]
fn test_no_max_gap_size_fills_everything() {
let values = vec![1.0, f64::NAN, f64::NAN, f64::NAN, 5.0];
let ts = ts_from(&values);
let filled = GapFiller::fill_gaps(&ts, GapFillMethod::LinearInterpolation, None).unwrap();
let out = pixel_series(&filled);
for (idx, v) in out.iter().enumerate() {
assert!(!v.is_nan(), "position {idx} should be filled, got NaN");
}
assert!((out[2] - 3.0).abs() < 1e-9);
}
#[test]
fn test_max_gap_size_zero_blocks_all_fills() {
let values = vec![1.0, f64::NAN, 3.0];
let ts = ts_from(&values);
let params = GapFillParams {
max_gap_size: Some(0),
..GapFillParams::default()
};
let filled =
GapFiller::fill_gaps(&ts, GapFillMethod::LinearInterpolation, Some(params)).unwrap();
let out = pixel_series(&filled);
assert!(out[1].is_nan(), "max_gap_size=0 must block all fills");
}
}