pub struct SyncConfig {Show 15 fields
pub access_key: Option<String>,
pub bucket: String,
pub checkers: Option<i64>,
pub endpoint: Option<String>,
pub fail_if_no_checkpoint: bool,
pub flags: Option<Vec<String>>,
pub ignore_checksum: Option<bool>,
pub multi_thread_cutoff: Option<String>,
pub multi_thread_streams: Option<i64>,
pub provider: Option<String>,
pub region: Option<String>,
pub secret_key: Option<String>,
pub start_from_checkpoint: Option<StartFromCheckpoint>,
pub transfers: Option<i64>,
pub upload_concurrency: Option<i64>,
}Expand description
SyncConfig
JSON schema
{
"type": "object",
"required": [
"bucket"
],
"properties": {
"access_key": {
"description": "The access key used to authenticate with the storage provider.\n\nIf not provided, rclone will fall back to environment-based credentials, such as\n`RCLONE_S3_ACCESS_KEY_ID`. In Kubernetes environments using IRSA (IAM Roles for Service Accounts),\nthis can be left empty to allow automatic authentication via the pod's service account.",
"type": [
"string",
"null"
]
},
"bucket": {
"description": "The name of the storage bucket.\n\nThis may include a path to a folder inside the bucket (e.g., `my-bucket/data`).",
"type": "string"
},
"checkers": {
"description": "The number of checkers to run in parallel.\nDefault: 20",
"type": [
"integer",
"null"
],
"format": "int32",
"minimum": 0.0
},
"endpoint": {
"description": "The endpoint URL for the storage service.\n\nThis is typically required for custom or local S3-compatible storage providers like MinIO.\nExample: `http://localhost:9000`\n\nRelevant rclone config key: [`endpoint`](https://rclone.org/s3/#s3-endpoint)",
"type": [
"string",
"null"
]
},
"fail_if_no_checkpoint": {
"description": "When true, the pipeline will fail to initialize if fetching the\nspecified checkpoint fails (missing, download error).\nWhen false, the pipeline will start from scratch instead.\n\nFalse by default.",
"default": false,
"type": "boolean"
},
"flags": {
"description": "Extra flags to pass to `rclone`.\n\nWARNING: Supplying incorrect or conflicting flags can break `rclone`.\nUse with caution.\n\nRefer to the docs to see the supported flags:\n- [Global flags](https://rclone.org/flags/)\n- [S3 specific flags](https://rclone.org/s3/)",
"type": [
"array",
"null"
],
"items": {
"type": "string"
}
},
"ignore_checksum": {
"description": "Set to skip post copy check of checksums, and only check the file sizes.\nThis can significantly improve the throughput.\nDefualt: false",
"type": [
"boolean",
"null"
]
},
"multi_thread_cutoff": {
"description": "Use multi-thread download for files above this size.\nFormat: `[size][Suffix]` (Example: 1G, 500M)\nSupported suffixes: k|M|G|T\nDefault: 100M",
"type": [
"string",
"null"
]
},
"multi_thread_streams": {
"description": "Number of streams to use for multi-thread downloads.\nDefault: 10",
"type": [
"integer",
"null"
],
"format": "int32",
"minimum": 0.0
},
"provider": {
"description": "The name of the cloud storage provider (e.g., `\"AWS\"`, `\"Minio\"`).\n\nUsed for provider-specific behavior in rclone.\nIf omitted, defaults to `\"Other\"`.\n\nSee [rclone S3 provider documentation](https://rclone.org/s3/#s3-provider)",
"type": [
"string",
"null"
]
},
"region": {
"description": "The region that this bucket is in.\n\nLeave empty for Minio or the default region (`us-east-1` for AWS).",
"type": [
"string",
"null"
]
},
"secret_key": {
"description": "The secret key used together with the access key for authentication.\n\nIf not provided, rclone will fall back to environment-based credentials, such as\n`RCLONE_S3_SECRET_ACCESS_KEY`. In Kubernetes environments using IRSA (IAM Roles for Service Accounts),\nthis can be left empty to allow automatic authentication via the pod's service account.",
"type": [
"string",
"null"
]
},
"start_from_checkpoint": {
"allOf": [
{
"$ref": "#/components/schemas/StartFromCheckpoint"
}
]
},
"transfers": {
"description": "The number of file transfers to run in parallel.\nDefault: 20",
"type": [
"integer",
"null"
],
"format": "int32",
"minimum": 0.0
},
"upload_concurrency": {
"description": "The number of chunks of the same file that are uploaded for multipart uploads.\nDefault: 10",
"type": [
"integer",
"null"
],
"format": "int32",
"minimum": 0.0
}
}
}Fields§
§access_key: Option<String>The access key used to authenticate with the storage provider.
If not provided, rclone will fall back to environment-based credentials, such as
RCLONE_S3_ACCESS_KEY_ID. In Kubernetes environments using IRSA (IAM Roles for Service Accounts),
this can be left empty to allow automatic authentication via the pod’s service account.
bucket: StringThe name of the storage bucket.
This may include a path to a folder inside the bucket (e.g., my-bucket/data).
checkers: Option<i64>The number of checkers to run in parallel. Default: 20
endpoint: Option<String>The endpoint URL for the storage service.
This is typically required for custom or local S3-compatible storage providers like MinIO.
Example: http://localhost:9000
Relevant rclone config key: endpoint
fail_if_no_checkpoint: boolWhen true, the pipeline will fail to initialize if fetching the specified checkpoint fails (missing, download error). When false, the pipeline will start from scratch instead.
False by default.
flags: Option<Vec<String>>Extra flags to pass to rclone.
WARNING: Supplying incorrect or conflicting flags can break rclone.
Use with caution.
Refer to the docs to see the supported flags:
ignore_checksum: Option<bool>Set to skip post copy check of checksums, and only check the file sizes. This can significantly improve the throughput. Defualt: false
multi_thread_cutoff: Option<String>Use multi-thread download for files above this size.
Format: [size][Suffix] (Example: 1G, 500M)
Supported suffixes: k|M|G|T
Default: 100M
multi_thread_streams: Option<i64>Number of streams to use for multi-thread downloads. Default: 10
provider: Option<String>The name of the cloud storage provider (e.g., "AWS", "Minio").
Used for provider-specific behavior in rclone.
If omitted, defaults to "Other".
region: Option<String>The region that this bucket is in.
Leave empty for Minio or the default region (us-east-1 for AWS).
secret_key: Option<String>The secret key used together with the access key for authentication.
If not provided, rclone will fall back to environment-based credentials, such as
RCLONE_S3_SECRET_ACCESS_KEY. In Kubernetes environments using IRSA (IAM Roles for Service Accounts),
this can be left empty to allow automatic authentication via the pod’s service account.
start_from_checkpoint: Option<StartFromCheckpoint>§transfers: Option<i64>The number of file transfers to run in parallel. Default: 20
upload_concurrency: Option<i64>The number of chunks of the same file that are uploaded for multipart uploads. Default: 10
Implementations§
Source§impl SyncConfig
impl SyncConfig
pub fn builder() -> SyncConfig
Trait Implementations§
Source§impl Clone for SyncConfig
impl Clone for SyncConfig
Source§fn clone(&self) -> SyncConfig
fn clone(&self) -> SyncConfig
1.0.0 · Source§fn clone_from(&mut self, source: &Self)
fn clone_from(&mut self, source: &Self)
source. Read more