copyrite 0.3.0

A CLI tool for efficient checksum and copy operations across object stores
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

//! Implementations for reading data using IO and from cloud storage.
//!

use crate::checksum::file::SumsFile;
use crate::error::{ApiError, Result};
use crate::io::Provider;
use crate::io::S3Client;
use crate::io::sums::aws::S3Builder;
use crate::io::sums::file::FileBuilder;
use dyn_clone::DynClone;
use futures_util::Stream;
use std::collections::HashSet;
use std::pin::Pin;
use std::sync::Arc;
use tokio::io::AsyncRead;

pub mod aws;
pub mod channel;
pub mod file;

/// The type returned when converting a shared reader into a stream.
pub type ReaderStream = Pin<Box<dyn Stream<Item = Result<Arc<[u8]>>> + Send>>;

/// The shared reader trait defines functions for accessing chunks of data from a
/// reader in a parallel context.
#[async_trait::async_trait]
pub trait SharedReader {
    /// Start the IO-based read task, which reads chunks of data from a reader
    /// until the end.
    async fn read_chunks(&mut self) -> Result<u64>;

    /// Convert the shared reader into a stream of the resulting bytes of reading
    /// the chunks.
    fn as_stream(&mut self) -> ReaderStream;
}

/// Read operations on file based or cloud sums files.
#[async_trait::async_trait]
pub trait ObjectSums: DynClone {
    /// Get an existing sums file for this object.
    async fn sums_file(&mut self) -> Result<Option<SumsFile>>;

    /// Get a reader to the sums files.
    async fn reader(&mut self) -> Result<Box<dyn AsyncRead + Unpin + Send>>;

    /// Get the file size of the target file.
    async fn file_size(&mut self) -> Result<Option<u64>>;

    /// Write data to the configured location.
    async fn write_sums_file(&self, sums_file: &SumsFile) -> Result<()>;

    /// Get the location of the object.
    fn location(&self) -> String;

    /// Any accumulated recoverable api errors.
    fn api_errors(&self) -> HashSet<ApiError>;
}

dyn_clone::clone_trait_object!(ObjectSums);

/// Build object sums from object URLs.
#[derive(Debug, Default)]
pub struct ObjectSumsBuilder {
    client: Option<S3Client>,
}

impl ObjectSumsBuilder {
    pub async fn build(self, url: String) -> Result<Box<dyn ObjectSums + Send>> {
        match Provider::try_from(url.as_str())? {
            Provider::File { file } => {
                Ok(Box::new(FileBuilder::default().with_file(file).build()?))
            }
            Provider::S3 { bucket, key } => {
                let client = self.client.ok_or_else(|| {
                    crate::error::Error::ParseError(
                        "an S3 client is required for S3 providers".to_string(),
                    )
                })?;
                Ok(Box::new(
                    S3Builder::default()
                        .with_key(key)
                        .with_bucket(bucket)
                        .with_client(client)
                        .build()?,
                ))
            }
        }
    }

    /// Set the S3 client if this is an s3 provider.
    pub fn set_client(mut self, client: Option<S3Client>) -> Self {
        self.client = client;
        self
    }
}