sqlload 0.66.0

SQL data loading tooling.
/// A file is written out once its rows reach this many.
pub const FILE_MAX_ROWS: usize = 1_000_000;

/// A file is written out once its encoded bytes reach about this many (128 MiB).
pub const FILE_MAX_BYTES: usize = 128 * 1024 * 1024;

/// The largest file is written out while all open files together hold more than this (512 MiB).
pub const OPEN_FILES_MAX_BYTES: usize = 512 * 1024 * 1024;

/// The name every written file starts with, followed by the load id and a part number.
pub const PART_PREFIX: &str = "part-";

/// The file name suffix of the Parquet files written.
pub const PARQUET_SUFFIX: &str = ".parquet";

/// The placeholder for the cursor's value in a path template.
pub const CURSOR_PLACEHOLDER: &str = "{cursor}";

/// The placeholder for the cursor's date in a path template.
pub const CURSOR_DATE_PLACEHOLDER: &str = "{cursor:date}";

/// The placeholder for the table's name in a path template.
pub const TABLE_PLACEHOLDER: &str = "{table}";

/// What every cursor placeholder starts with.
pub const CURSOR_PLACEHOLDER_START: &str = "{cursor";

/// The time of day partition bounds of timestamp cursors are written with.
pub const MIDNIGHT: &str = "00:00:00.000000";

/// The offset zoned timestamp bounds end with.
pub const UTC_SUFFIX: &str = "+00";

/// Seconds in a day, to turn Parquet date statistics (days since 1970) into dates.
pub const SECONDS_PER_DAY: i64 = 86_400;

/// The folder below a table's folder holding, per commit of a files source's load, the source files whose rows that commit's files hold.
pub const LOADED_FOLDER: &str = "_sqlload/loaded/";

/// The suffix of a commit's record in [`LOADED_FOLDER`], which no data file has.
pub const LOADED_SUFFIX: &str = ".json";

/// The folder below a table's folder holding one marker per load that has not finished.
pub const PENDING_FOLDER: &str = "_sqlload/pending/";

/// The folder below a table's folder holding how each commit of a load ended: `replacing` once the load committed its files, `aborted` once another load repaired it. Each is created only if absent, so of a load committing late and a load repairing it, exactly one decides.
pub const DECIDED_FOLDER: &str = "_sqlload/decided/";

/// The most a `single_file` commit holds in memory across its open files (2 GiB); each file is stored in one put at the commit, well under S3's 5 GiB limit for one put.
pub const SINGLE_FILE_MAX_BYTES: usize = 2 * 1024 * 1024 * 1024;

/// Marks the hidden folder a filesystem `replace` stages the table in, beside the table folder: `.<table folder>.sqlload-<load id>`.
pub const STAGED_FOLDER_MARK: &str = ".sqlload-";

/// The length of a load id, a hyphenated UUID, as staging folder names end with it.
pub const LOAD_ID_LENGTH: usize = 36;

/// Added to a staging folder's name by releases before 0.63.0 for the old table they moved aside; a stopped load could leave one, which loads keep and warn about.
pub const MOVED_ASIDE_SUFFIX: &str = "-old";

/// The first line of a pending marker while the load writes its files.
pub const PENDING_WRITING: &str = "writing";

/// The first line of a pending marker while the load deletes the files it replaces.
pub const PENDING_REPLACING: &str = "replacing";

/// The first line of the marker repair leaves for a load it found writing: the load may only have lost its lease and still be writing, so any file it stores later is deleted too.
pub const PENDING_ABORTED: &str = "aborted";

/// How long repair keeps an `aborted` marker (7 days), well past any lease a load still writing could have held.
pub const ABORTED_KEPT_MS: i64 = 7 * 24 * 60 * 60 * 1000;

/// At most this many partitions have a file open at once; beyond it the largest is written out.
pub const OPEN_FILES_MAX: usize = 256;

/// Memory an open file holds beyond its buffered data (writer state, page buffers), for the memory estimate.
pub const OPEN_FILE_OVERHEAD_BYTES: usize = 1024 * 1024;

/// The start of a pending marker line holding a run record (keys never start with `@`).
pub const PENDING_RUN_PREFIX: &str = "@run\t";

/// Records of loaded source files read at once while planning.
pub const RECORD_READS: usize = 16;

/// The start of a record's name that merges the records of several commits, which no commit's own record starts with (load ids are UUIDs).
pub const COMPACTED_PREFIX: &str = "compacted.";

/// A load merges a table's live records into one once there are more than this many.
pub const RECORDS_COMPACTED_ABOVE: usize = 64;

/// The file name suffix of the CSV files written, before any compression suffix.
pub const CSV_SUFFIX: &str = ".csv";

/// The suffix of gzip-compressed CSV files.
pub const GZIP_SUFFIX: &str = ".gz";

/// The suffix of zstd-compressed CSV files.
pub const ZSTD_SUFFIX: &str = ".zst";

/// The name suffixes of a files table's data files, whatever format wrote them, so a replace removes files of an earlier format too.
pub const DATA_SUFFIXES: &[&str] = &[".parquet", ".csv", ".csv.gz", ".csv.zst"];

/// CSV text compressed into one zstd frame once this many bytes wait (4 MiB).
pub const ZSTD_FRAME_BYTES: usize = 4 * 1024 * 1024;

/// The file name of a `single_file` table whose folder has no name of its own.
pub const SINGLE_FILE_STEM: &str = "data";

/// How timestamps with a zone are written to CSV: RFC 3339 in UTC.
pub const CSV_TIMESTAMP_TZ: &str = "%Y-%m-%dT%H:%M:%S%.6fZ";

/// How timestamps without a zone are written to CSV.
pub const CSV_TIMESTAMP: &str = "%Y-%m-%dT%H:%M:%S%.6f";

/// How dates are written to CSV.
pub const CSV_DATE: &str = "%Y-%m-%d";

/// How times of day are written to CSV.
pub const CSV_TIME: &str = "%H:%M:%S%.6f";