pub mod args;
use crate::callback::event_manager::EventManager;
use crate::callback::filter_manager::FilterManager;
use crate::types::{ClientConfigLocation, S3Credentials, StoragePath};
use aws_sdk_s3::types::RequestPayer;
use aws_smithy_types::checksum_config::RequestChecksumCalculation;
use chrono::{DateTime, Utc};
use fancy_regex::Regex;
#[derive(Debug, Clone)]
pub struct Config {
pub target: StoragePath,
pub show_no_progress: bool,
pub target_client_config: Option<ClientConfig>,
pub force_retry_config: ForceRetryConfig,
pub tracing_config: Option<TracingConfig>,
pub worker_size: u16,
pub warn_as_error: bool,
pub dry_run: bool,
pub rate_limit_objects: Option<u32>,
pub max_parallel_listings: u16,
pub object_listing_queue_size: u32,
pub max_parallel_listing_max_depth: u16,
pub allow_parallel_listings_in_express_one_zone: bool,
pub filter_config: FilterConfig,
pub max_keys: i32,
pub auto_complete_shell: Option<clap_complete::shells::Shell>,
pub event_callback_lua_script: Option<String>,
pub filter_callback_lua_script: Option<String>,
pub allow_lua_os_library: bool,
pub allow_lua_unsafe_vm: bool,
pub lua_vm_memory_limit: usize,
pub lua_callback_timeout_milliseconds: u64,
pub if_match: bool,
pub max_delete: Option<u64>,
pub filter_manager: FilterManager,
pub event_manager: EventManager,
pub batch_size: u16,
pub delete_all_versions: bool,
pub force: bool,
pub test_user_defined_callback: bool,
}
impl Config {
pub fn for_target(bucket: &str, prefix: &str) -> Self {
Config {
target: StoragePath::S3 {
bucket: bucket.to_string(),
prefix: prefix.to_string(),
},
force: true,
..Config::default()
}
}
}
impl Default for Config {
fn default() -> Self {
Config {
target: StoragePath::S3 {
bucket: String::new(),
prefix: String::new(),
},
show_no_progress: false,
target_client_config: None,
force_retry_config: ForceRetryConfig::default(),
tracing_config: None,
worker_size: 16,
warn_as_error: false,
dry_run: false,
rate_limit_objects: None,
max_parallel_listings: 16,
object_listing_queue_size: 200_000,
max_parallel_listing_max_depth: 2,
allow_parallel_listings_in_express_one_zone: false,
filter_config: FilterConfig::default(),
max_keys: 1000,
auto_complete_shell: None,
event_callback_lua_script: None,
filter_callback_lua_script: None,
allow_lua_os_library: false,
allow_lua_unsafe_vm: false,
lua_vm_memory_limit: 64 * 1024 * 1024,
lua_callback_timeout_milliseconds: 10_000,
if_match: false,
max_delete: None,
filter_manager: FilterManager::new(),
event_manager: EventManager::new(),
batch_size: 200,
delete_all_versions: false,
force: false,
test_user_defined_callback: false,
}
}
}
impl Default for ForceRetryConfig {
fn default() -> Self {
ForceRetryConfig {
force_retry_count: 0,
force_retry_interval_milliseconds: 1000,
}
}
}
#[derive(Debug, Clone)]
pub struct ClientConfig {
pub client_config_location: ClientConfigLocation,
pub credential: S3Credentials,
pub region: Option<String>,
pub endpoint_url: Option<String>,
pub force_path_style: bool,
pub accelerate: bool,
pub request_payer: Option<RequestPayer>,
pub retry_config: RetryConfig,
pub cli_timeout_config: CLITimeoutConfig,
pub disable_stalled_stream_protection: bool,
pub request_checksum_calculation: RequestChecksumCalculation,
}
#[derive(Debug, Clone)]
pub struct RetryConfig {
pub aws_max_attempts: u32,
pub initial_backoff_milliseconds: u64,
}
#[derive(Debug, Clone)]
pub struct CLITimeoutConfig {
pub operation_timeout_milliseconds: Option<u64>,
pub operation_attempt_timeout_milliseconds: Option<u64>,
pub connect_timeout_milliseconds: Option<u64>,
pub read_timeout_milliseconds: Option<u64>,
}
#[derive(Debug, Clone, Copy)]
pub struct TracingConfig {
pub tracing_level: log::Level,
pub json_tracing: bool,
pub aws_sdk_tracing: bool,
pub span_events_tracing: bool,
pub disable_color_tracing: bool,
}
#[derive(Debug, Clone, Copy)]
pub struct ForceRetryConfig {
pub force_retry_count: u32,
pub force_retry_interval_milliseconds: u64,
}
#[derive(Debug, Clone, Default)]
pub struct FilterConfig {
pub before_time: Option<DateTime<Utc>>,
pub after_time: Option<DateTime<Utc>>,
pub include_regex: Option<Regex>,
pub exclude_regex: Option<Regex>,
pub include_content_type_regex: Option<Regex>,
pub exclude_content_type_regex: Option<Regex>,
pub include_metadata_regex: Option<Regex>,
pub exclude_metadata_regex: Option<Regex>,
pub include_tag_regex: Option<Regex>,
pub exclude_tag_regex: Option<Regex>,
pub larger_size: Option<u64>,
pub smaller_size: Option<u64>,
pub keep_latest_only: bool,
pub delete_marker_only: bool,
}
#[cfg(test)]
mod tests {
use super::*;
use crate::test_utils::init_dummy_tracing_subscriber;
#[test]
fn retry_config_creation() {
init_dummy_tracing_subscriber();
let retry_config = RetryConfig {
aws_max_attempts: 3,
initial_backoff_milliseconds: 100,
};
assert_eq!(retry_config.aws_max_attempts, 3);
assert_eq!(retry_config.initial_backoff_milliseconds, 100);
}
#[test]
fn cli_timeout_config_creation() {
init_dummy_tracing_subscriber();
let timeout_config = CLITimeoutConfig {
operation_timeout_milliseconds: Some(30000),
operation_attempt_timeout_milliseconds: Some(10000),
connect_timeout_milliseconds: Some(5000),
read_timeout_milliseconds: Some(5000),
};
assert_eq!(timeout_config.operation_timeout_milliseconds, Some(30000));
assert_eq!(
timeout_config.operation_attempt_timeout_milliseconds,
Some(10000)
);
}
#[test]
fn cli_timeout_config_no_timeouts() {
init_dummy_tracing_subscriber();
let timeout_config = CLITimeoutConfig {
operation_timeout_milliseconds: None,
operation_attempt_timeout_milliseconds: None,
connect_timeout_milliseconds: None,
read_timeout_milliseconds: None,
};
assert!(timeout_config.operation_timeout_milliseconds.is_none());
}
#[test]
fn tracing_config_creation() {
init_dummy_tracing_subscriber();
let tracing_config = TracingConfig {
tracing_level: log::Level::Info,
json_tracing: false,
aws_sdk_tracing: false,
span_events_tracing: false,
disable_color_tracing: false,
};
assert_eq!(tracing_config.tracing_level, log::Level::Info);
assert!(!tracing_config.json_tracing);
}
#[test]
fn force_retry_config_creation() {
init_dummy_tracing_subscriber();
let force_retry = ForceRetryConfig {
force_retry_count: 3,
force_retry_interval_milliseconds: 1000,
};
assert_eq!(force_retry.force_retry_count, 3);
assert_eq!(force_retry.force_retry_interval_milliseconds, 1000);
}
#[test]
fn filter_config_default() {
init_dummy_tracing_subscriber();
let filter_config = FilterConfig::default();
assert!(filter_config.before_time.is_none());
assert!(filter_config.after_time.is_none());
assert!(filter_config.include_regex.is_none());
assert!(filter_config.exclude_regex.is_none());
assert!(filter_config.larger_size.is_none());
assert!(filter_config.smaller_size.is_none());
}
#[test]
fn config_for_target_sets_bucket_and_prefix() {
init_dummy_tracing_subscriber();
let config = Config::for_target("my-bucket", "logs/2024/");
let StoragePath::S3 { bucket, prefix } = &config.target;
assert_eq!(bucket, "my-bucket");
assert_eq!(prefix, "logs/2024/");
}
#[test]
fn config_for_target_sets_force_true() {
let config = Config::for_target("bucket", "prefix/");
assert!(config.force);
}
#[test]
fn config_for_target_uses_default_worker_and_batch_size() {
let config = Config::for_target("bucket", "");
assert_eq!(config.worker_size, 16);
assert_eq!(config.batch_size, 200);
}
#[test]
fn config_for_target_has_sensible_defaults() {
let config = Config::for_target("bucket", "prefix/");
assert!(!config.dry_run);
assert!(!config.delete_all_versions);
assert!(!config.if_match);
assert!(config.max_delete.is_none());
assert!(config.tracing_config.is_none());
assert!(config.target_client_config.is_none());
assert!(config.rate_limit_objects.is_none());
assert!(!config.warn_as_error);
assert!(!config.test_user_defined_callback);
}
#[test]
fn config_default_has_empty_target() {
let config = Config::default();
let StoragePath::S3 { bucket, prefix } = &config.target;
assert!(bucket.is_empty());
assert!(prefix.is_empty());
}
#[test]
fn config_default_does_not_set_force() {
let config = Config::default();
assert!(!config.force);
}
#[test]
fn config_default_field_values() {
let config = Config::default();
assert_eq!(config.worker_size, 16);
assert_eq!(config.batch_size, 200);
assert!(!config.show_no_progress);
assert!(!config.dry_run);
assert!(!config.warn_as_error);
assert_eq!(config.max_parallel_listings, 16);
assert_eq!(config.object_listing_queue_size, 200_000);
assert_eq!(config.max_parallel_listing_max_depth, 2);
assert!(!config.allow_parallel_listings_in_express_one_zone);
assert_eq!(config.max_keys, 1000);
assert!(config.auto_complete_shell.is_none());
assert!(config.event_callback_lua_script.is_none());
assert!(config.filter_callback_lua_script.is_none());
assert!(!config.allow_lua_os_library);
assert!(!config.allow_lua_unsafe_vm);
assert_eq!(config.lua_vm_memory_limit, 64 * 1024 * 1024);
assert_eq!(config.lua_callback_timeout_milliseconds, 10_000);
assert!(!config.if_match);
assert!(!config.delete_all_versions);
}
#[test]
fn force_retry_config_default_values() {
let frc = ForceRetryConfig::default();
assert_eq!(frc.force_retry_count, 0);
assert_eq!(frc.force_retry_interval_milliseconds, 1000);
}
}