pub const DEFAULT_CONSUMER_GROUP: &str = "sqlload-{stream}";
pub const DEFAULT_DISCOVERY_INTERVAL_SECS: u64 = 300;
pub const DEFAULT_PREFETCH_BUDGET: u64 = 1024 * 1024 * 1024;
pub const DEFAULT_STARTUP_BATCH: usize = 20;
pub const DEFAULT_STARTUP_INTERVAL_MS: u64 = 500;
pub const DEFAULT_MAX_IN_FLIGHT: usize = 32;
pub const DEFAULT_MAX_PENDING_FILES: usize = 32;
pub const DEFAULT_PENDING_CHECK_AFTER_SECS: u64 = 5 * 60;
pub const DEFAULT_PENDING_RECHECK_EVERY_SECS: u64 = 10 * 60;
pub const DEFAULT_RETAIN_SECS: u64 = 7 * 24 * 3600;
pub const DEFAULT_MAX_BYTES: u64 = 32 * 1024 * 1024;
pub const DEFAULT_MAX_ROWS: usize = 100_000;
pub const DEFAULT_MAX_AGE_SECS: u64 = 60;
pub const DEFAULT_MAX_TOTAL_BYTES: u64 = 2 * 1024 * 1024 * 1024;
pub const DEFAULT_MAX_REQUEST_BYTES: u64 = 3_500_000;
pub const DEFAULT_STAGE_TEMPLATE: &str = "{table}_SQLLOAD_STAGE";
pub const DEFAULT_PIPE_TEMPLATE: &str = "{table}_SQLLOAD_PIPE";
pub const DEFAULT_STREAMING_PIPE_TEMPLATE: &str = "{table}-STREAMING";
pub const STREAMING_REQUEST_LIMIT: u64 = 4 * 1024 * 1024;
pub const MAX_NAME_LENGTH: usize = 255;
pub const SIZE_UNITS: &[(&str, f64)] = &[
("", 1.0),
("b", 1.0),
("k", 1e3),
("kb", 1e3),
("m", 1e6),
("mb", 1e6),
("g", 1e9),
("gb", 1e9),
("kib", 1024.0),
("mib", 1_048_576.0),
("gib", 1_073_741_824.0),
];
pub const DEFAULT_S3_PATH: &str = "{topic}/date={date}/";
pub const S3_PATH_PLACEHOLDERS: &[&str] = &["topic", "table", "stream", "date", "hour"];
pub const ENVIRONMENT_PREFIX: &str = "ENV:";
pub const SNOWFLAKE_COLUMNS: &[(&str, &str, &str)] = &[
("TOPIC", "topic", "VARCHAR"),
("KAFKA_TIMESTAMP", "timestamp", "NUMBER(38,0)"),
("KAFKA_TIMESTAMP_TYPE", "timestamp_type", "NUMBER(38,0)"),
("KAFKA_OFFSET", "offset", "NUMBER(38,0)"),
("KAFKA_PARTITION", "partition", "NUMBER(38,0)"),
("KEY", "key", "VARCHAR"),
("VALUE", "value", "VARCHAR"),
("EXPORTED_AT", "exported_at", "TIMESTAMP_TZ"),
("DATE", "date", "DATE"),
];
pub const DATABRICKS_COLUMNS: &[(&str, &str, &str)] = &[
("topic", "topic", "STRING"),
("kafka_timestamp", "timestamp", "BIGINT"),
("kafka_timestamp_type", "timestamp_type", "INT"),
("kafka_offset", "offset", "BIGINT"),
("kafka_partition", "partition", "INT"),
("key", "key", "STRING"),
("value", "value", "STRING"),
("exported_at", "exported_at", "TIMESTAMP"),
("date", "date", "DATE"),
];