#![cfg(e2e_test)]
#[cfg(test)]
mod common;
#[cfg(test)]
mod tests {
use common::*;
use s3sync::config::Config;
use s3sync::config::args::parse_from_args;
use s3sync::pipeline::Pipeline;
use s3sync::types::token::create_pipeline_cancellation_token;
use std::convert::TryFrom;
use super::*;
#[tokio::test]
async fn test_lua_event_callback_no_script() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--event-callback-lua-script",
"./test_data/script/event_callback.luaERROR",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let args = parse_from_args(args);
assert!(args.is_err());
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_event_callback() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--event-callback-lua-script",
"./test_data/script/event_callback.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(!pipeline.has_error());
let stats = TestHelper::get_stats_count(pipeline.get_stats_receiver());
assert_eq!(stats.sync_complete, 5);
assert_eq!(stats.e_tag_verified, 5);
assert_eq!(stats.sync_skip, 0);
assert_eq!(stats.checksum_verified, 0);
assert_eq!(stats.sync_warning, 0);
}
{
let target_bucket_url = format!("s3://{}", bucket);
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--delete",
"--event-callback-lua-script",
"./test_data/script/event_callback.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(!pipeline.has_error());
let stats = TestHelper::get_stats_count(pipeline.get_stats_receiver());
assert_eq!(stats.sync_complete, 0);
assert_eq!(stats.e_tag_verified, 0);
assert_eq!(stats.sync_skip, 5);
assert_eq!(stats.checksum_verified, 0);
assert_eq!(stats.sync_warning, 0);
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_event_callback_versioning() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket1 = TestHelper::generate_bucket_name();
let bucket2 = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket1);
helper.create_bucket(&bucket1, REGION).await;
helper.create_bucket(&bucket2, REGION).await;
helper.enable_bucket_versioning(&bucket1).await;
helper.enable_bucket_versioning(&bucket2).await;
helper.sync_test_data(&target_bucket_url).await;
}
let source_bucket_url = format!("s3://{}", bucket1);
let target_bucket_url = format!("s3://{}", bucket2);
{
let args = vec![
"s3sync",
"--source-profile",
"s3sync-e2e-test",
"--target-profile",
"s3sync-e2e-test",
"--server-side-copy",
"--enable-versioning",
&source_bucket_url,
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(!pipeline.has_error());
let object_versions_list = helper.list_object_versions(&bucket2, "").await;
assert_eq!(object_versions_list.len(), 5);
}
{
helper.delete_all_objects(&bucket1).await;
}
{
let args = vec![
"s3sync",
"--source-profile",
"s3sync-e2e-test",
"--target-profile",
"s3sync-e2e-test",
"--server-side-copy",
"--enable-versioning",
&source_bucket_url,
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(!pipeline.has_error());
let object_versions_list = helper.list_object_versions(&bucket2, "").await;
assert_eq!(object_versions_list.len(), 5);
let object_list = helper.list_objects(&bucket2, "").await;
assert_eq!(object_list.len(), 0);
}
{
let target_bucket_url = format!("s3://{}", bucket1);
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--delete",
"./test_data/e2e_test/case2/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(!pipeline.has_error());
}
{
let args = vec![
"s3sync",
"--source-profile",
"s3sync-e2e-test",
"--target-profile",
"s3sync-e2e-test",
"--server-side-copy",
"--enable-versioning",
"--event-callback-lua-script",
"./test_data/script/event_callback.lua",
&source_bucket_url,
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(!pipeline.has_error());
let object_versions_list = helper.list_object_versions(&bucket2, "").await;
assert_eq!(object_versions_list.len(), 9);
let object_list = helper.list_objects(&bucket2, "").await;
assert_eq!(object_list.len(), 4);
}
helper.delete_bucket_with_cascade(&bucket1).await;
helper.delete_bucket_with_cascade(&bucket2).await;
}
#[tokio::test]
async fn test_lua_event_callback_twice() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--event-callback-lua-script",
"./test_data/script/event_callback.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(!pipeline.has_error());
let stats = TestHelper::get_stats_count(pipeline.get_stats_receiver());
assert_eq!(stats.sync_complete, 5);
assert_eq!(stats.e_tag_verified, 5);
assert_eq!(stats.sync_skip, 0);
assert_eq!(stats.checksum_verified, 0);
assert_eq!(stats.sync_warning, 0);
}
{
let target_bucket_url = format!("s3://{}", bucket);
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--event-callback-lua-script",
"./test_data/script/event_callback.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(!pipeline.has_error());
let stats = TestHelper::get_stats_count(pipeline.get_stats_receiver());
assert_eq!(stats.sync_complete, 0);
assert_eq!(stats.e_tag_verified, 0);
assert_eq!(stats.sync_skip, 5);
assert_eq!(stats.checksum_verified, 0);
assert_eq!(stats.sync_warning, 0);
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_event_callback_error() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--event-callback-lua-script",
"./test_data/script/event_callback_error.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(!pipeline.has_error());
let stats = TestHelper::get_stats_count(pipeline.get_stats_receiver());
assert_eq!(stats.sync_complete, 5);
assert_eq!(stats.e_tag_verified, 5);
assert_eq!(stats.sync_skip, 0);
assert_eq!(stats.checksum_verified, 0);
assert_eq!(stats.sync_warning, 0);
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_event_callback_no_entrypoint() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--event-callback-lua-script",
"./test_data/script/no_entrypoint.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(!pipeline.has_error());
let stats = TestHelper::get_stats_count(pipeline.get_stats_receiver());
assert_eq!(stats.sync_complete, 5);
assert_eq!(stats.e_tag_verified, 5);
assert_eq!(stats.sync_skip, 0);
assert_eq!(stats.checksum_verified, 0);
assert_eq!(stats.sync_warning, 0);
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_filter_callback_no_script() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--filter-callback-lua-script",
"./test_data/script/filter_callback.luaERROR",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let args = parse_from_args(args);
assert!(args.is_err());
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_filter_callback() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--event-callback-lua-script",
"./test_data/script/event_callback.lua",
"--filter-callback-lua-script",
"./test_data/script/filter_callback.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(!pipeline.has_error());
let stats = TestHelper::get_stats_count(pipeline.get_stats_receiver());
assert_eq!(stats.sync_complete, 1);
assert_eq!(stats.e_tag_verified, 1);
assert_eq!(stats.sync_skip, 4);
assert_eq!(stats.checksum_verified, 0);
assert_eq!(stats.sync_warning, 0);
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_filter_callback_no_entrypoint() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--filter-callback-lua-script",
"./test_data/script/no_entrypoint.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(pipeline.has_error());
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_filter_callback_with_os_lib_error() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--filter-callback-lua-script",
"./test_data/script/filter_callback_with_os_lib.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(pipeline.has_error());
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_filter_callback_with_io_lib_error() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--filter-callback-lua-script",
"./test_data/script/filter_callback_with_io_lib.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(pipeline.has_error());
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_filter_callback_with_io_lib() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--filter-callback-lua-script",
"./test_data/script/filter_callback_with_io_lib.lua",
"--allow-lua-os-library",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(!pipeline.has_error());
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_filter_callback_with_error() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--filter-callback-lua-script",
"./test_data/script/filter_callback_error.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(pipeline.has_error());
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_filter_callback_with_os_lib() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--filter-callback-lua-script",
"./test_data/script/filter_callback_with_os_lib.lua",
"--allow-lua-os-library",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(!pipeline.has_error());
let stats = TestHelper::get_stats_count(pipeline.get_stats_receiver());
assert_eq!(stats.sync_complete, 1);
assert_eq!(stats.e_tag_verified, 1);
assert_eq!(stats.sync_skip, 4);
assert_eq!(stats.checksum_verified, 0);
assert_eq!(stats.sync_warning, 0);
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_filter_callback_with_unsafe_lua_vm() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--filter-callback-lua-script",
"./test_data/script/filter_callback_with_os_lib.lua",
"--allow-lua-unsafe-vm",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(!pipeline.has_error());
let stats = TestHelper::get_stats_count(pipeline.get_stats_receiver());
assert_eq!(stats.sync_complete, 1);
assert_eq!(stats.e_tag_verified, 1);
assert_eq!(stats.sync_skip, 4);
assert_eq!(stats.checksum_verified, 0);
assert_eq!(stats.sync_warning, 0);
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_preprocess_callback_no_script() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--preprocess-callback-lua-script",
"./test_data/script/preprocess_callback.luaERROR",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let args = parse_from_args(args);
assert!(args.is_err());
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_preprocess_callback_no_entrypoint() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--preprocess-callback-lua-script",
"./test_data/script/no_entrypoint.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(pipeline.has_error());
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_preprocess_callback() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--preprocess-callback-lua-script",
"./test_data/script/preprocess_callback.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(!pipeline.has_error());
let stats = TestHelper::get_stats_count(pipeline.get_stats_receiver());
assert_eq!(stats.sync_complete, 5);
assert_eq!(stats.e_tag_verified, 5);
assert_eq!(stats.sync_skip, 0);
assert_eq!(stats.checksum_verified, 0);
assert_eq!(stats.sync_warning, 0);
helper
.verify_test_object_metadata(&bucket, "dir1/data1", None)
.await;
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_preprocess_callback_all_metadata() {
TestHelper::init_dummy_tracing_subscriber();
const TEST_PREFIX: &str = "mydir";
let helper = TestHelper::new().await;
let bucket1 = TestHelper::generate_bucket_name();
let bucket2 = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}/{}/", bucket1, TEST_PREFIX);
helper.create_bucket(&bucket1, REGION).await;
helper.create_bucket(&bucket2, REGION).await;
helper.sync_test_data(&target_bucket_url).await;
}
let source_bucket_url = format!("s3://{}/{}/", bucket1, TEST_PREFIX);
let target_bucket_url = format!("s3://{}/{}/", bucket2, TEST_PREFIX);
{
let args = vec![
"s3sync",
"--source-profile",
"s3sync-e2e-test",
"--target-profile",
"s3sync-e2e-test",
"--storage-class",
"REDUCED_REDUNDANCY",
"--acl",
"bucket-owner-full-control",
"--content-disposition",
TEST_CONTENT_DISPOSITION,
"--content-encoding",
TEST_CONTENT_ENCODING,
"--content-language",
TEST_CONTENT_LANGUAGE,
"--cache-control",
TEST_CACHE_CONTROL,
"--content-type",
TEST_CONTENT_TYPE,
"--expires",
TEST_EXPIRES,
"--metadata",
TEST_METADATA_STRING,
"--tagging",
TEST_TAGGING,
"--website-redirect",
TEST_WEBSITE_REDIRECT2,
"--target-request-payer",
"--preprocess-callback-lua-script",
"./test_data/script/preprocess_callback.lua",
&source_bucket_url,
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(!pipeline.has_error());
let stats = TestHelper::get_stats_count(pipeline.get_stats_receiver());
assert_eq!(stats.sync_complete, 5);
assert_eq!(stats.e_tag_verified, 5);
assert_eq!(stats.sync_skip, 0);
assert_eq!(stats.checksum_verified, 0);
assert_eq!(stats.sync_warning, 0);
}
helper.delete_bucket_with_cascade(&bucket1).await;
helper.delete_bucket_with_cascade(&bucket2).await;
}
#[tokio::test]
async fn test_lua_preprocess_callback_multipart_upload() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
TestHelper::create_large_file();
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--preprocess-callback-lua-script",
"./test_data/script/preprocess_callback.lua",
LARGE_FILE_DIR,
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(!pipeline.has_error());
let stats = TestHelper::get_stats_count(pipeline.get_stats_receiver());
assert_eq!(stats.sync_complete, 1);
assert_eq!(stats.e_tag_verified, 1);
assert_eq!(stats.sync_skip, 0);
assert_eq!(stats.checksum_verified, 0);
assert_eq!(stats.sync_warning, 0);
helper
.verify_test_object_metadata(&bucket, LARGE_FILE_KEY, None)
.await;
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_preprocess_callback_invalid_acl() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--preprocess-callback-lua-script",
"./test_data/script/preprocess_callback_invalid_acl.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(pipeline.has_error());
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_preprocess_callback_invalid_expire() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--preprocess-callback-lua-script",
"./test_data/script/preprocess_callback_invalid_expire.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(pipeline.has_error());
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_preprocess_callback_invalid_storage_class() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--preprocess-callback-lua-script",
"./test_data/script/preprocess_callback_invalid_storage_class.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(pipeline.has_error());
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_preprocess_callback_invalid_request_payer() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--preprocess-callback-lua-script",
"./test_data/script/preprocess_callback_invalid_request_payer.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(pipeline.has_error());
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_preprocess_callback_skip() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--preprocess-callback-lua-script",
"./test_data/script/preprocess_callback_skip.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(!pipeline.has_error());
let stats = TestHelper::get_stats_count(pipeline.get_stats_receiver());
assert_eq!(stats.sync_complete, 0);
assert_eq!(stats.e_tag_verified, 0);
assert_eq!(stats.sync_skip, 5);
assert_eq!(stats.checksum_verified, 0);
assert_eq!(stats.sync_warning, 0);
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_invalid_script() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--preprocess-callback-lua-script",
"./test_data/script/invalid_script.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap());
assert!(config.is_err());
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_not_enough_memory() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--lua-vm-memory-limit",
"1KiB",
"--preprocess-callback-lua-script",
"./test_data/script/preprocess_callback.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap());
assert!(config.is_err());
}
helper.delete_bucket_with_cascade(&bucket).await;
}
#[tokio::test]
async fn test_lua_preprocess_callback_to_all_clear() {
TestHelper::init_dummy_tracing_subscriber();
let helper = TestHelper::new().await;
let bucket = TestHelper::generate_bucket_name();
{
let target_bucket_url = format!("s3://{}", bucket);
helper.create_bucket(&bucket, REGION).await;
let args = vec![
"s3sync",
"--target-profile",
"s3sync-e2e-test",
"--content-disposition",
TEST_CONTENT_DISPOSITION,
"--content-encoding",
TEST_CONTENT_ENCODING,
"--content-language",
TEST_CONTENT_LANGUAGE,
"--cache-control",
TEST_CACHE_CONTROL,
"--content-type",
TEST_CONTENT_TYPE,
"--expires",
TEST_EXPIRES,
"--metadata",
TEST_METADATA_STRING,
"--tagging",
TEST_TAGGING,
"--website-redirect",
TEST_WEBSITE_REDIRECT,
"--target-request-payer",
"--storage-class",
"STANDARD_IA",
"--acl",
"bucket-owner-full-control",
"--preprocess-callback-lua-script",
"./test_data/script/preprocess_callback_to_nil.lua",
"./test_data/e2e_test/case1/",
&target_bucket_url,
];
let config = Config::try_from(parse_from_args(args).unwrap()).unwrap();
let cancellation_token = create_pipeline_cancellation_token();
let mut pipeline = Pipeline::new(config.clone(), cancellation_token).await;
pipeline.run().await;
assert!(!pipeline.has_error());
let stats = TestHelper::get_stats_count(pipeline.get_stats_receiver());
assert_eq!(stats.sync_complete, 5);
assert_eq!(stats.e_tag_verified, 5);
assert_eq!(stats.sync_skip, 0);
assert_eq!(stats.checksum_verified, 0);
assert_eq!(stats.sync_warning, 0);
helper
.verify_test_object_no_metadata(&bucket, "dir1/data1", None)
.await;
}
helper.delete_bucket_with_cascade(&bucket).await;
}
}