coeus-ops 0.10.0

Numerical tensor operators for Coeus.
Documentation
use crate::ptr::{MutPtr, Ptr};
use coeus_core::{Backend, CpuAddressableStorage, CpuAddressableStorageMut, Layout, Scalar};

#[inline]
pub(crate) fn avg_pool2d<T: Scalar, B: Backend>(
    backend: &B,
    input: &B::DeviceBuffer<T>,
    input_layout: &Layout,
    kernel_size: usize,
    stride: usize,
    padding: usize,
    dilation: usize,
    output: &mut B::DeviceBuffer<T>,
    output_layout: &Layout,
) where
    B::DeviceBuffer<T>: CpuAddressableStorageMut<T>,
{
    let n = input_layout.shape()[0];
    let c = input_layout.shape()[1];
    let h = input_layout.shape()[2];
    let w = input_layout.shape()[3];
    let h_out = output_layout.shape()[2];
    let w_out = output_layout.shape()[3];
    let out_numel = n * c * h_out * w_out;

    let input_slice = input.as_slice();
    let output_slice = output.as_mut_slice();

    let input_ptr = Ptr(input_slice.as_ptr());
    let output_ptr = MutPtr(output_slice.as_mut_ptr());

    let input_layout = input_layout.clone();
    let output_layout = output_layout.clone();

    let pad_s = padding as isize;
    let stride_s = stride as isize;
    let dil_s = dilation as isize;
    let k_size = kernel_size;

    backend.parallel_for(0, out_numel, move |i| {
        let ow = i % w_out;
        let temp1 = i / w_out;
        let oh = temp1 % h_out;
        let temp2 = temp1 / h_out;
        let ci = temp2 % c;
        let ni = temp2 / c;

        let mut sum = T::zero();
        let mut count = 0usize;

        for ikh in 0..k_size {
            let h_in = oh as isize * stride_s + ikh as isize * dil_s - pad_s;
            if h_in >= 0 && (h_in as usize) < h {
                for ikw in 0..k_size {
                    let w_in = ow as isize * stride_s + ikw as isize * dil_s - pad_s;
                    if w_in >= 0 && (w_in as usize) < w {
                        let input_idx =
                            input_layout.physical_index(&[ni, ci, h_in as usize, w_in as usize]);
                        let val = unsafe { input_ptr.read(input_idx) };
                        sum += val;
                        count += 1;
                    }
                }
            }
        }

        let mean = if count > 0 {
            sum / T::from_f64(count as f64)
        } else {
            T::zero()
        };

        let output_idx = output_layout.physical_index(&[ni, ci, oh, ow]);
        unsafe {
            output_ptr.write(output_idx, mean);
        }
    });
}

#[inline]
pub(crate) fn avg_pool2d_backward<T: Scalar, B: Backend>(
    backend: &B,
    grad_out: &B::DeviceBuffer<T>,
    grad_out_layout: &Layout,
    kernel_size: usize,
    stride: usize,
    padding: usize,
    dilation: usize,
    grad_input: &mut B::DeviceBuffer<T>,
    grad_input_layout: &Layout,
) where
    B::DeviceBuffer<T>: CpuAddressableStorageMut<T>,
{
    let n = grad_input_layout.shape()[0];
    let c = grad_input_layout.shape()[1];
    let h = grad_input_layout.shape()[2];
    let w = grad_input_layout.shape()[3];
    let h_out = grad_out_layout.shape()[2];
    let w_out = grad_out_layout.shape()[3];
    let numel_in = n * c * h * w;

    let go_slice = grad_out.as_slice();
    let gi_slice = grad_input.as_mut_slice();

    let go_ptr = Ptr(go_slice.as_ptr());
    let gi_ptr = MutPtr(gi_slice.as_mut_ptr());

    let go_layout = grad_out_layout.clone();
    let gi_layout = grad_input_layout.clone();

    let pad_s = padding as isize;
    let stride_s = stride as isize;
    let dil_s = dilation as isize;
    let k_size = kernel_size;

    backend.parallel_for(0, numel_in, move |i| {
        let wi = i % w;
        let temp1 = i / w;
        let hi = temp1 % h;
        let temp2 = temp1 / h;
        let ci = temp2 % c;
        let ni = temp2 / c;

        let mut sum = T::zero();

        for ikh in 0..k_size {
            let numer_h = hi as isize + pad_s - ikh as isize * dil_s;
            if numer_h >= 0 && numer_h % stride_s == 0 {
                let oh = (numer_h / stride_s) as usize;
                if oh < h_out {
                    for ikw in 0..k_size {
                        let numer_w = wi as isize + pad_s - ikw as isize * dil_s;
                        if numer_w >= 0 && numer_w % stride_s == 0 {
                            let ow = (numer_w / stride_s) as usize;
                            if ow < w_out {
                                let mut count = 0usize;
                                for jkh in 0..k_size {
                                    let h_in =
                                        oh as isize * stride_s + jkh as isize * dil_s - pad_s;
                                    if h_in >= 0 && (h_in as usize) < h {
                                        for jkw in 0..k_size {
                                            let w_in = ow as isize * stride_s
                                                + jkw as isize * dil_s
                                                - pad_s;
                                            if w_in >= 0 && (w_in as usize) < w {
                                                count += 1;
                                            }
                                        }
                                    }
                                }
                                if count > 0 {
                                    let go_idx = go_layout.physical_index(&[ni, ci, oh, ow]);
                                    let gval = unsafe { go_ptr.read(go_idx) };
                                    sum += gval / T::from_f64(count as f64);
                                }
                            }
                        }
                    }
                }
            }
        }

        let gi_idx = gi_layout.physical_index(&[ni, ci, hi, wi]);
        unsafe {
            let old = gi_ptr.read(gi_idx);
            gi_ptr.write(gi_idx, old + sum);
        }
    });
}

#[inline]
pub(crate) fn avg_pool3d<T: Scalar, B: Backend>(
    backend: &B,
    input: &B::DeviceBuffer<T>,
    input_layout: &Layout,
    kernel_size: usize,
    stride: usize,
    padding: usize,
    dilation: usize,
    output: &mut B::DeviceBuffer<T>,
    output_layout: &Layout,
) where
    B::DeviceBuffer<T>: CpuAddressableStorageMut<T>,
{
    let n = input_layout.shape()[0];
    let c = input_layout.shape()[1];
    let d = input_layout.shape()[2];
    let h = input_layout.shape()[3];
    let w = input_layout.shape()[4];
    let d_out = output_layout.shape()[2];
    let h_out = output_layout.shape()[3];
    let w_out = output_layout.shape()[4];
    let out_numel = n * c * d_out * h_out * w_out;

    let input_slice = input.as_slice();
    let output_slice = output.as_mut_slice();

    let input_ptr = Ptr(input_slice.as_ptr());
    let output_ptr = MutPtr(output_slice.as_mut_ptr());

    let input_layout = input_layout.clone();
    let output_layout = output_layout.clone();

    let pad_s = padding as isize;
    let stride_s = stride as isize;
    let dil_s = dilation as isize;
    let k_size = kernel_size;

    backend.parallel_for(0, out_numel, move |i| {
        let ow = i % w_out;
        let temp1 = i / w_out;
        let oh = temp1 % h_out;
        let temp2 = temp1 / h_out;
        let od = temp2 % d_out;
        let temp3 = temp2 / d_out;
        let ci = temp3 % c;
        let ni = temp3 / c;

        let mut sum = T::zero();
        let mut count = 0usize;

        for ikd in 0..k_size {
            let d_in = od as isize * stride_s + ikd as isize * dil_s - pad_s;
            if d_in >= 0 && (d_in as usize) < d {
                for ikh in 0..k_size {
                    let h_in = oh as isize * stride_s + ikh as isize * dil_s - pad_s;
                    if h_in >= 0 && (h_in as usize) < h {
                        for ikw in 0..k_size {
                            let w_in = ow as isize * stride_s + ikw as isize * dil_s - pad_s;
                            if w_in >= 0 && (w_in as usize) < w {
                                let input_idx = input_layout.physical_index(&[
                                    ni,
                                    ci,
                                    d_in as usize,
                                    h_in as usize,
                                    w_in as usize,
                                ]);
                                let val = unsafe { input_ptr.read(input_idx) };
                                sum += val;
                                count += 1;
                            }
                        }
                    }
                }
            }
        }

        let mean = if count > 0 {
            sum / T::from_f64(count as f64)
        } else {
            T::zero()
        };

        let output_idx = output_layout.physical_index(&[ni, ci, od, oh, ow]);
        unsafe {
            output_ptr.write(output_idx, mean);
        }
    });
}

#[inline]
pub(crate) fn avg_pool3d_backward<T: Scalar, B: Backend>(
    backend: &B,
    grad_out: &B::DeviceBuffer<T>,
    grad_out_layout: &Layout,
    kernel_size: usize,
    stride: usize,
    padding: usize,
    dilation: usize,
    grad_input: &mut B::DeviceBuffer<T>,
    grad_input_layout: &Layout,
) where
    B::DeviceBuffer<T>: CpuAddressableStorageMut<T>,
{
    let n = grad_input_layout.shape()[0];
    let c = grad_input_layout.shape()[1];
    let d = grad_input_layout.shape()[2];
    let h = grad_input_layout.shape()[3];
    let w = grad_input_layout.shape()[4];
    let d_out = grad_out_layout.shape()[2];
    let h_out = grad_out_layout.shape()[3];
    let w_out = grad_out_layout.shape()[4];
    let numel_in = n * c * d * h * w;

    let go_slice = grad_out.as_slice();
    let gi_slice = grad_input.as_mut_slice();

    let go_ptr = Ptr(go_slice.as_ptr());
    let gi_ptr = MutPtr(gi_slice.as_mut_ptr());

    let go_layout = grad_out_layout.clone();
    let gi_layout = grad_input_layout.clone();

    let pad_s = padding as isize;
    let stride_s = stride as isize;
    let dil_s = dilation as isize;
    let k_size = kernel_size;

    backend.parallel_for(0, numel_in, move |i| {
        let wi = i % w;
        let temp1 = i / w;
        let hi = temp1 % h;
        let temp2 = temp1 / h;
        let di = temp2 % d;
        let temp3 = temp2 / d;
        let ci = temp3 % c;
        let ni = temp3 / c;

        let mut sum = T::zero();

        for ikd in 0..k_size {
            let numer_d = di as isize + pad_s - ikd as isize * dil_s;
            if numer_d >= 0 && numer_d % stride_s == 0 {
                let od = (numer_d / stride_s) as usize;
                if od < d_out {
                    for ikh in 0..k_size {
                        let numer_h = hi as isize + pad_s - ikh as isize * dil_s;
                        if numer_h >= 0 && numer_h % stride_s == 0 {
                            let oh = (numer_h / stride_s) as usize;
                            if oh < h_out {
                                for ikw in 0..k_size {
                                    let numer_w = wi as isize + pad_s - ikw as isize * dil_s;
                                    if numer_w >= 0 && numer_w % stride_s == 0 {
                                        let ow = (numer_w / stride_s) as usize;
                                        if ow < w_out {
                                            let mut count = 0usize;
                                            for jkd in 0..k_size {
                                                let d_in = od as isize * stride_s
                                                    + jkd as isize * dil_s
                                                    - pad_s;
                                                if d_in >= 0 && (d_in as usize) < d {
                                                    for jkh in 0..k_size {
                                                        let h_in = oh as isize * stride_s
                                                            + jkh as isize * dil_s
                                                            - pad_s;
                                                        if h_in >= 0 && (h_in as usize) < h {
                                                            for jkw in 0..k_size {
                                                                let w_in = ow as isize * stride_s
                                                                    + jkw as isize * dil_s
                                                                    - pad_s;
                                                                if w_in >= 0 && (w_in as usize) < w
                                                                {
                                                                    count += 1;
                                                                }
                                                            }
                                                        }
                                                    }
                                                }
                                            }
                                            if count > 0 {
                                                let go_idx =
                                                    go_layout.physical_index(&[ni, ci, od, oh, ow]);
                                                let gval = unsafe { go_ptr.read(go_idx) };
                                                sum += gval / T::from_f64(count as f64);
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }

        let gi_idx = gi_layout.physical_index(&[ni, ci, di, hi, wi]);
        unsafe {
            let old = gi_ptr.read(gi_idx);
            gi_ptr.write(gi_idx, old + sum);
        }
    });
}