s3ueeze 0.1.0

#![deny(clippy::unwrap_used)]

mod metric;
mod worker;

use anyhow::{Result, anyhow, bail};
use aws_config::BehaviorVersion;
use aws_sdk_s3::Client;
use buildstructor::buildstructor;
use metric::Metric;
use std::{fmt::Debug, num::NonZeroUsize, thread, time::Duration};
use tokio::{
    fs::OpenOptions,
    io::{AsyncWriteExt, BufWriter},
    select, spawn,
    sync::mpsc::{self},
    time::interval,
};
use tokio_util::sync::CancellationToken;
#[cfg(feature = "tracing")]
use tracing::{debug, info, instrument, trace};
use uom::{
    fmt::DisplayStyle,
    si::{
        f64::{Information, InformationRate, Time},
        information::{byte, mebibyte},
        information_rate::megabit_per_second,
        time::second,
    },
};
use url::Url;
use worker::Worker;

#[derive(Clone, Debug)]
pub struct S3ueeze {
    client: Client,
    destination: Url,
    sources: Vec<Url>,
    worker_count: usize,
}

#[buildstructor]
impl S3ueeze {
    /// # Errors
    ///
    /// Returns an error if unable to read number of available cores,
    /// multiplying the cores by our constant overflows, or the S3 URL is
    /// malformed.
    #[builder]
    #[cfg_attr(feature = "tracing", instrument)]
    pub async fn new(
        client: Option<Client>,
        sources: Vec<Url>,
        destination: Url,
        worker_count: Option<NonZeroUsize>,
    ) -> Result<Self> {
        let client = if let Some(client) = client {
            client
        } else {
            let config =
                aws_config::load_defaults(BehaviorVersion::latest()).await;
            Client::new(&config)
        };

        for url in &sources {
            if url.scheme() != "s3" {
                bail!("source schemes must all be s3://: {url:?}");
            }

            if url.host().is_none() {
                bail!(
                    "source hosts must all be valid s3 bucket names: {url:?}"
                );
            }
        }

        if destination.scheme() != "file" {
            bail!("destination scheme must be path://: {destination:?}");
        }

        let worker_count = if let Some(workers) = worker_count {
            workers.get()
        } else {
            thread::available_parallelism()?
                .get()
                .checked_mul(5)
                .ok_or_else(|| {
                    anyhow!("multiplying available parallelism overflowed")
                })?
        };

        Ok(Self {
            client,
            destination,
            sources,
            worker_count,
        })
    }

    /// Fetch the number of workers `S3ueeze` will use.
    #[must_use]
    pub fn worker_count(&self) -> usize {
        self.worker_count
    }

    /// # Errors
    ///
    /// Returns an error in the following situations:
    ///
    /// 1. failure to open the destination file for writing and truncate it;
    /// 2. failure to send a source S3 URI to a worker for processing;
    /// 3. other I/O errors writing to the file;
    /// 4. any issues connecting to S3.
    //
    // TODO: Use a dedicated error type.
    #[allow(clippy::too_many_lines)]
    #[cfg_attr(feature = "tracing", instrument(err, skip_all))]
    pub async fn run(
        &self,
        cancel: CancellationToken,
    ) -> Result<(u64, u64, u64)> {
        // Create channels for all the kinds of responses we need from the workers.
        let (objects_tx, mut objects_rx) = mpsc::unbounded_channel();
        let (bytes_tx, mut bytes_rx) = mpsc::unbounded_channel();

        // Collection of worker channels.
        let mut workers = vec![];

        // Go through and spawn workers connected to all the channels.
        for n in 0..self.worker_count {
            #[cfg(feature = "tracing")]
            trace!("spawning worker {n}");

            // Create channels for all the requests the workers need to receive.
            // Make a new channel on which the worker will receive messages.
            let (sources_tx, sources_rx) = mpsc::unbounded_channel();
            let (contents_tx, contents_rx) = mpsc::unbounded_channel();

            // Construct a worker.
            let worker = Worker::builder()
                .client(self.client.clone())
                .objects_tx(objects_tx.clone())
                .bytes_tx(bytes_tx.clone())
                .cancel(cancel.clone())
                .build();

            // Spawn the async worker, storing its join handle and channels.
            workers.push((
                spawn(
                    async move { worker.work(sources_rx, contents_rx).await },
                ),
                sources_tx,
                contents_tx,
            ));
        }
        #[cfg(feature = "tracing")]
        debug!("done spawning {} workers", workers.len());

        // Open a file for writing the output.
        let f = OpenOptions::new()
            .write(true)
            .create(true)
            .truncate(true)
            .open(self.destination.path())
            .await?;
        // Put the file into a BufWriter for buffered output.
        let mut f = BufWriter::new(f);

        // Zip the sources and workers together cyclically so that every source
        // gets assigned to a worker, then iterate over the pairs.
        for (source, (n, (_, tx, _))) in
            self.sources.iter().zip(workers.iter().enumerate().cycle())
        {
            #[cfg(feature = "tracing")]
            debug!("sending source {source} to worker {n}");
            tx.send(source.clone())?;
        }

        // Set up a ticker for status updates.
        let mut ticker = interval(Duration::from_secs(5));

        // Total processed sources.
        let mut processed_sources = 0;

        // Total processed bytes.
        let mut processed_bytes: Metric<u64> = Metric::default();

        // Total processed objects.
        let mut total_objects: u64 = 0;

        // Total processed bytes.
        // TODO: Use uom for typed Information.
        let mut total_bytes: u64 = 0;

        // Total processed lines.
        let mut processed_lines = 0;

        // Keep running our event loop, handling data as it flows in and out.
        loop {
            select! {
                // Use a biased select, so we favor, in order:
                // 1. cancellation;
                // 2. emitting stats according to our stats ticker;
                // 3. receiving a line of JSON for writing to the buffer;
                // 4. receiving a list of objects from a bucket/path and sending
                //    them back out to a worker to be read.
                biased;

                () = cancel.cancelled() => {
                    // If our token's been cancelled, exit the loop.
                    break;
                }
                _ = ticker.tick() => {
                    // Make sure we have the latest metrics data.
                    let new = processed_bytes.flush();
                    if new < 1 && processed_bytes.total()? > 0 {
                        cancel.cancel();

                        break;
                    }

                    let bytes = Information::new::<byte>(new as f64);

                    // Calculate Mbps.
                    let speed: InformationRate = (
                        bytes / Time::new::<second>(5.0)
                    ).into();
                    let speed = speed
                        .into_format_args(
                            megabit_per_second,
                            DisplayStyle::Abbreviation,
                        );
                    let bytes = bytes
                        .into_format_args(
                            mebibyte,
                            DisplayStyle::Abbreviation,
                        );

                        let percent = if total_bytes > 0 {
                            processed_bytes.total()? as f64 / total_bytes as f64 * 100.0
                        } else {
                            0.0
                        };

                    eprintln!("{percent:.1}%, {speed}");
                    #[cfg(feature = "tracing")]
                    info!(
                        %speed,
                        %processed_lines,
                        %processed_sources,
                        %total_objects,
                        processed_bytes = processed_bytes.total()?,
                        %bytes,
                    );

                    if self.sources.len() == processed_sources
                        && total_bytes == processed_bytes.total()?
                    {
                        #[cfg(feature = "tracing")]
                        info!("done; shutting down workers");

                        // Shut down the workers cleanly.
                        cancel.cancel();

                        break;
                    }
                }
                // Receive a line of JSON (these are JSON-object-per-line
                // files) and write it out to disk, collecting metrics along
                // the way.
                Some(line) = bytes_rx.recv() => {
                    // Write the whole line to the file.
                    f.write_all(line.as_bytes()).await?;
                    // Write a newline.
                    f.write_u8(b'\n').await?;

                    // Add the written line + the newline to the total bytes,
                    // and increment the line counter.
                    processed_bytes.add(line.len() as u64 +1);
                    processed_lines += 1;
                }
                // Receive a list of objects from a bucket and fan them out to
                // workers to fetch.
                Some((bucket, objects)) = objects_rx.recv() => {
                    #[cfg(feature = "tracing")]
                    debug!(?bucket, objects = ?objects.len());

                    // Zip together all the objects in the payload with the
                    // workers, cyclically, so each object gets assigned to a
                    // worker and we saturate the workers.
                    for (
                        object,
                        (n, (_, _, contents_tx))
                    ) in objects.into_iter().zip(workers.iter().enumerate().cycle())
                    {
                        #[cfg(feature = "tracing")]
                        trace!("sending {:?} to worker {n}", object.key());

                        let Some(size) = object.size() else {
                            bail!("object missing size: object key: {:?}", object.key());
                        };

                        // Send the bucket and object to a worker.
                        contents_tx.send((
                            bucket.clone(),
                            object,
                        ))?;


                        total_bytes += u64::try_from(size)?;
                        total_objects += 1;
                    }

                    processed_sources += 1;
                },
            }
        }

        // Go through and shut down each worker, checking the return value.
        for (handle, _, _) in workers {
            handle.await??;
        }

        // Make sure our stats are up to date.
        processed_bytes.flush();

        // Return the metadata about what we did.
        Ok((total_objects, processed_bytes.total()?, processed_lines))
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    macro_rules! url {
        ($str:expr) => {
            Url::try_from($str).unwrap()
        };
    }

    #[tokio::test]
    async fn test_config() {
        // Base case.
        S3ueeze::builder()
            .destination(url!("file:///tmp/foobar"))
            .build()
            .await
            .unwrap();

        // With one source.
        S3ueeze::builder()
            .source(url!("s3://bucket/tmp/foobar"))
            .destination(url!("file:///tmp/foobar"))
            .build()
            .await
            .unwrap();

        // With multiple sources.
        S3ueeze::builder()
            .sources(vec![
                url!("s3://bucket/tmp/foobar"),
                url!("s3://bucket/tmp/foobar"),
            ])
            .destination(url!("file:///tmp/foobar"))
            .build()
            .await
            .unwrap();
    }
}