timelake_format/lib.rs
1//! Timelake's on-disk conventions, with no I/O of its own, so the engine (`timelake`) and the
2//! WebAssembly reader and writer (`timelake-wasm`) share one definition: the table contract stored
3//! in `timelake.*` properties, time partition math, and Iceberg bound decoding. With the `write`
4//! feature, also the write path: preparing rows, writing Parquet with Iceberg statistics,
5//! manifests and table metadata, and appending and committing over a [`host::Host`] that the
6//! caller implements (SPEC §21.10).
7//!
8//! ```
9//! use std::collections::HashMap;
10//! use timelake_format::{Chunk, Contract, property};
11//!
12//! let properties = HashMap::from([
13//! (property::TIME_COLUMN.to_string(), "time".to_string()),
14//! (property::CHUNK.to_string(), "day".to_string()),
15//! ]);
16//! let contract = Contract::from_properties(&properties)?;
17//! assert_eq!(contract.chunk, Some(Chunk::Day));
18//!
19//! let day = Chunk::Day.value(1_767_225_600_000_000)?; // 2026-01-01T00:00:00Z in µs
20//! assert_eq!(Chunk::Day.label(day), "2026-01-01");
21//! # Ok::<(), timelake_format::FormatError>(())
22//! ```
23
24#![warn(missing_docs)]
25
26use std::fmt;
27
28#[cfg(feature = "write")]
29pub mod append;
30pub mod bounds;
31#[cfg(feature = "write")]
32pub mod commit;
33pub mod contract;
34#[cfg(feature = "write")]
35pub mod error;
36#[cfg(feature = "write")]
37pub mod file;
38#[cfg(feature = "head")]
39pub mod head;
40#[cfg(feature = "write")]
41pub mod host;
42#[cfg(feature = "write")]
43pub mod iceberg;
44#[cfg(feature = "write")]
45pub mod maintain;
46#[cfg(feature = "write")]
47pub mod merge;
48#[cfg(feature = "write")]
49pub mod metadata;
50#[cfg(feature = "write")]
51pub mod rows;
52pub mod settings;
53#[cfg(feature = "write")]
54pub mod table;
55pub mod version;
56
57pub use contract::{Chunk, Contract, TableOptions, format_duration, parse_duration};
58pub use version::FormatVersion;
59
60/// Table property keys holding the [`Contract`].
61pub mod property {
62 /// The timestamp column.
63 pub const TIME_COLUMN: &str = "timelake.time-column";
64 /// Comma-separated series key columns.
65 pub const SERIES_KEY: &str = "timelake.series-key";
66 /// Time partition size: `hour`, `day` or `month`.
67 pub const CHUNK: &str = "timelake.chunk-interval";
68 /// `true` to drop duplicate `(series, time)` rows during compaction.
69 pub const DEDUP: &str = "timelake.dedup";
70 /// Data retention, e.g. `90d`.
71 pub const RETENTION: &str = "timelake.retention";
72 /// Target size of compacted files, in bytes.
73 pub const TARGET_FILE_SIZE: &str = "timelake.target-file-size-bytes";
74 /// Rows per Parquet row group.
75 pub const ROW_GROUP_ROWS: &str = "timelake.row-group-rows";
76 /// `false` to skip inline maintenance on append.
77 pub const AUTO_MAINTAIN: &str = "timelake.auto-maintain";
78 /// How long to keep old snapshots, e.g. `1h`.
79 pub const SNAPSHOT_RETENTION: &str = "timelake.snapshot-retention";
80 /// Snapshots always kept.
81 pub const MIN_SNAPSHOTS: &str = "timelake.min-snapshots";
82 /// Snapshots kept at most.
83 pub const MAX_SNAPSHOTS: &str = "timelake.max-snapshots";
84 /// Files per size tier that trigger inline compaction.
85 pub const COMPACT_MIN_FILES: &str = "timelake.compact-min-files";
86 /// Inline maintenance rewrites a partition whole (sorted, duplicates removed) once the table
87 /// holds data newer than the partition's end by this much, or `off`.
88 pub const COMPACT_COLD_AFTER: &str = "timelake.compact-cold-after";
89 /// The column holding each row's insertion time (`_ingest_ts`), set by the writer; absent for
90 /// tables without one. Among rows with the same `(series…, time)` the latest insertion wins.
91 pub const INGEST_TIME_COLUMN: &str = "timelake.ingest-time-column";
92 /// The Timelake reader removes copies of a `(series…, time)` when reading (default: as `dedup`).
93 pub const READ_DEDUP: &str = "timelake.read-dedup";
94 /// How the Timelake reader removes copies: `merge` (default) or `keys-first` (SPEC §9.1).
95 pub const READ_DEDUP_STRATEGY: &str = "timelake.read-dedup-strategy";
96 /// Whether the Timelake reader filters rows while decoding Parquet: `auto` (default), `true` or `false`.
97 pub const FILTER_PUSHDOWN: &str = "timelake.filter-pushdown";
98 /// The Timelake reader prunes row groups and pages with `IN` lists of more than 20 values.
99 pub const IN_LIST_PRUNING: &str = "timelake.in-list-pruning";
100 /// `ORDER BY time DESC LIMIT k` reads time partitions newest first and stops early.
101 pub const NEWEST_FIRST: &str = "timelake.newest-first";
102 /// How `time_weighted_mean` gets readings in time order: `auto` (default), `segments` or `merge`.
103 pub const TIME_WEIGHTED_MEAN: &str = "timelake.time-weighted-mean";
104 /// Format of the head object (`metadata/timelake-head`): `arrow` (default), `json`, another
105 /// registered codec, or `none` to not write one.
106 pub const HEAD_FORMAT: &str = "timelake.head-format";
107 /// Bytes the head may spend on optional sections such as the last-value index.
108 pub const HEAD_INLINE_BYTES: &str = "timelake.head-inline-bytes";
109 /// No snapshot expires before this time (milliseconds since the epoch). Set by `timelake upgrade`
110 /// so the table can be rolled back to its state before the upgrade.
111 pub const KEEP_SNAPSHOTS_UNTIL: &str = "timelake.keep-snapshots-until";
112 /// The table format version (see [`crate::FORMAT_VERSION`]); absent means 0.1.0.
113 pub const FORMAT_VERSION: &str = "timelake.format-version";
114 /// On a raw table: its rollups, comma-separated table names (SPEC §11.1).
115 pub const ROLLUPS: &str = "timelake.rollups";
116 /// `false` to stop the Timelake reader answering queries on this table from its rollups.
117 pub const ROLLUP_ROUTING: &str = "timelake.rollup-routing";
118 /// On a rollup: the raw table it summarises.
119 pub const ROLLUP_SOURCE: &str = "timelake.rollup-source";
120 /// On a rollup: the bucket width, e.g. `1m`.
121 pub const ROLLUP_BUCKET: &str = "timelake.rollup-bucket";
122 /// On a rollup: its aggregates, as JSON (`[{"name": "avg_value", "function": "avg", "column": "value"}]`).
123 pub const ROLLUP_AGGREGATES: &str = "timelake.rollup-aggregates";
124 /// On a rollup: inline maintenance of the raw table refreshes the rollup once its last refresh
125 /// is this old, or `off`.
126 pub const ROLLUP_REFRESH_INTERVAL: &str = "timelake.rollup-refresh-interval";
127 /// How long the Timelake reader uses a table version without checking for a newer one.
128 pub const VERSION_CHECK_INTERVAL: &str = "timelake.version-check-interval";
129 /// Rows per Parquet data page in the files written.
130 pub const DATA_PAGE_ROWS: &str = "timelake.data-page-rows";
131 /// Iceberg's Parquet codec: `zstd` (default), `lz4`, `snappy`, `gzip`, `brotli`, or `uncompressed`.
132 pub const COMPRESSION_CODEC: &str = "write.parquet.compression-codec";
133 /// Iceberg's Parquet compression level, for `zstd`, `gzip` and `brotli`.
134 pub const COMPRESSION_LEVEL: &str = "write.parquet.compression-level";
135}
136
137/// Snapshot summary keys: Timelake's own, and the standard Iceberg counters Timelake maintains.
138pub mod summary {
139 /// Location of the snapshot's last-value index.
140 pub const LAST_VALUE_INDEX: &str = "timelake.last-value-index";
141 /// The writer-supplied idempotency key of an append.
142 pub const BATCH_ID: &str = "timelake.batch-id";
143 /// On a rollup: the id of the raw table snapshot the rollup reflects.
144 pub const ROLLUP_SOURCE_SNAPSHOT: &str = "timelake.rollup-source-snapshot";
145 /// On a rollup: the sequence number of that snapshot. Data files of the raw table with a higher
146 /// data sequence number hold rows the rollup doesn't reflect yet.
147 pub const ROLLUP_SOURCE_SEQUENCE: &str = "timelake.rollup-source-sequence";
148
149 /// Data files added by the snapshot.
150 pub const ADDED_DATA_FILES: &str = "added-data-files";
151 /// Rows added by the snapshot.
152 pub const ADDED_RECORDS: &str = "added-records";
153 /// Bytes added by the snapshot.
154 pub const ADDED_FILES_SIZE: &str = "added-files-size";
155 /// Data files removed by the snapshot.
156 pub const DELETED_DATA_FILES: &str = "deleted-data-files";
157 /// Rows removed by the snapshot.
158 pub const DELETED_RECORDS: &str = "deleted-records";
159 /// Bytes removed by the snapshot.
160 pub const REMOVED_FILES_SIZE: &str = "removed-files-size";
161 /// Data files in the table after the snapshot.
162 pub const TOTAL_DATA_FILES: &str = "total-data-files";
163 /// Rows in the table after the snapshot.
164 pub const TOTAL_RECORDS: &str = "total-records";
165 /// Bytes in the table after the snapshot.
166 pub const TOTAL_FILES_SIZE: &str = "total-files-size";
167 /// Delete files in the table (always 0: Timelake doesn't write them).
168 pub const TOTAL_DELETE_FILES: &str = "total-delete-files";
169 /// Position deletes in the table (always 0).
170 pub const TOTAL_POSITION_DELETES: &str = "total-position-deletes";
171 /// Equality deletes in the table (always 0).
172 pub const TOTAL_EQUALITY_DELETES: &str = "total-equality-deletes";
173}
174
175/// The table format this version of Timelake writes, recorded in each table as
176/// `timelake.format-version`. It reads and writes older formats, and newer ones as far as
177/// [`FormatVersion`] allows. `timelake upgrade` brings a table to this format.
178///
179/// - 0.1.0: tables written by Timelake 0.1 and 0.2 (no property).
180/// - 0.2.0: the head object (`metadata/timelake-head`), from Timelake 0.3 (which wrote it as `2`).
181/// - 0.3.0: an insertion-time column (`timelake.ingest-time-column`) that writers fill.
182pub const FORMAT_VERSION: FormatVersion = FormatVersion::new(0, 3, 0);
183
184/// The name of the insertion-time column of new time-series tables.
185pub const INGEST_TIME_COLUMN: &str = "_ingest_ts";
186
187/// Optional warehouse-level index of table names, for stores that cannot list (plain HTTPS).
188pub const TABLES_INDEX: &str = "timelake-tables.json";
189
190/// An invalid contract property, duration, or partition value.
191#[derive(Debug, Clone, PartialEq, Eq)]
192pub struct FormatError(pub String);
193
194impl FormatError {
195 fn new(msg: impl Into<String>) -> Self {
196 Self(msg.into())
197 }
198}
199
200impl fmt::Display for FormatError {
201 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
202 f.write_str(&self.0)
203 }
204}
205
206impl std::error::Error for FormatError {}
207
208/// `Result` with [`FormatError`].
209pub type Result<T> = std::result::Result<T, FormatError>;