timelake_format/lib.rs
1//! Timelake's on-disk conventions, with no I/O and no heavy dependencies, so the engine
2//! (`timelake`) and the WebAssembly reader (`timelake-wasm`) share one definition: the table
3//! contract stored in `timelake.*` properties, time partition math, and Iceberg bound decoding.
4//!
5//! ```
6//! use std::collections::HashMap;
7//! use timelake_format::{Chunk, Contract, property};
8//!
9//! let properties = HashMap::from([
10//! (property::TIME_COLUMN.to_string(), "time".to_string()),
11//! (property::CHUNK.to_string(), "day".to_string()),
12//! ]);
13//! let contract = Contract::from_properties(&properties)?;
14//! assert_eq!(contract.chunk, Some(Chunk::Day));
15//!
16//! let day = Chunk::Day.value(1_767_225_600_000_000)?; // 2026-01-01T00:00:00Z in µs
17//! assert_eq!(Chunk::Day.label(day), "2026-01-01");
18//! # Ok::<(), timelake_format::FormatError>(())
19//! ```
20
21#![warn(missing_docs)]
22
23use std::fmt;
24
25pub mod bounds;
26pub mod contract;
27#[cfg(feature = "head")]
28pub mod head;
29pub mod version;
30
31pub use contract::{Chunk, Contract, TableOptions, format_duration, parse_duration};
32pub use version::FormatVersion;
33
34/// Table property keys holding the [`Contract`].
35pub mod property {
36 /// The timestamp column.
37 pub const TIME_COLUMN: &str = "timelake.time-column";
38 /// Comma-separated series key columns.
39 pub const SERIES_KEY: &str = "timelake.series-key";
40 /// Time partition size: `hour`, `day` or `month`.
41 pub const CHUNK: &str = "timelake.chunk-interval";
42 /// `true` to drop duplicate `(series, time)` rows during compaction.
43 pub const DEDUP: &str = "timelake.dedup";
44 /// Data retention, e.g. `90d`.
45 pub const RETENTION: &str = "timelake.retention";
46 /// Target size of compacted files, in bytes.
47 pub const TARGET_FILE_SIZE: &str = "timelake.target-file-size-bytes";
48 /// Rows per Parquet row group.
49 pub const ROW_GROUP_ROWS: &str = "timelake.row-group-rows";
50 /// `false` to skip inline maintenance on append.
51 pub const AUTO_MAINTAIN: &str = "timelake.auto-maintain";
52 /// How long to keep old snapshots, e.g. `1h`.
53 pub const SNAPSHOT_RETENTION: &str = "timelake.snapshot-retention";
54 /// Snapshots always kept.
55 pub const MIN_SNAPSHOTS: &str = "timelake.min-snapshots";
56 /// Snapshots kept at most.
57 pub const MAX_SNAPSHOTS: &str = "timelake.max-snapshots";
58 /// Files per size tier that trigger inline compaction.
59 pub const COMPACT_MIN_FILES: &str = "timelake.compact-min-files";
60 /// Format of the head object (`metadata/timelake-head`): `arrow` (default), `json`, another
61 /// registered codec, or `none` to not write one.
62 pub const HEAD_FORMAT: &str = "timelake.head-format";
63 /// Bytes the head may spend on optional sections such as the last-value index.
64 pub const HEAD_INLINE_BYTES: &str = "timelake.head-inline-bytes";
65 /// No snapshot expires before this time (milliseconds since the epoch). Set by `timelake upgrade`
66 /// so the table can be rolled back to its state before the upgrade.
67 pub const KEEP_SNAPSHOTS_UNTIL: &str = "timelake.keep-snapshots-until";
68 /// The table format version (see [`crate::FORMAT_VERSION`]); absent means 0.1.0.
69 pub const FORMAT_VERSION: &str = "timelake.format-version";
70}
71
72/// Snapshot summary keys: Timelake's own, and the standard Iceberg counters Timelake maintains.
73pub mod summary {
74 /// Location of the snapshot's last-value index.
75 pub const LAST_VALUE_INDEX: &str = "timelake.last-value-index";
76 /// The writer-supplied idempotency key of an append.
77 pub const BATCH_ID: &str = "timelake.batch-id";
78
79 /// Data files added by the snapshot.
80 pub const ADDED_DATA_FILES: &str = "added-data-files";
81 /// Rows added by the snapshot.
82 pub const ADDED_RECORDS: &str = "added-records";
83 /// Bytes added by the snapshot.
84 pub const ADDED_FILES_SIZE: &str = "added-files-size";
85 /// Data files removed by the snapshot.
86 pub const DELETED_DATA_FILES: &str = "deleted-data-files";
87 /// Rows removed by the snapshot.
88 pub const DELETED_RECORDS: &str = "deleted-records";
89 /// Bytes removed by the snapshot.
90 pub const REMOVED_FILES_SIZE: &str = "removed-files-size";
91 /// Data files in the table after the snapshot.
92 pub const TOTAL_DATA_FILES: &str = "total-data-files";
93 /// Rows in the table after the snapshot.
94 pub const TOTAL_RECORDS: &str = "total-records";
95 /// Bytes in the table after the snapshot.
96 pub const TOTAL_FILES_SIZE: &str = "total-files-size";
97 /// Delete files in the table (always 0: Timelake doesn't write them).
98 pub const TOTAL_DELETE_FILES: &str = "total-delete-files";
99 /// Position deletes in the table (always 0).
100 pub const TOTAL_POSITION_DELETES: &str = "total-position-deletes";
101 /// Equality deletes in the table (always 0).
102 pub const TOTAL_EQUALITY_DELETES: &str = "total-equality-deletes";
103}
104
105/// The table format this version of Timelake writes, recorded in each table as
106/// `timelake.format-version`. It reads and writes older formats, and newer ones as far as
107/// [`FormatVersion`] allows. `timelake upgrade` brings a table to this format.
108///
109/// - 0.1.0: tables written by Timelake 0.1 and 0.2 (no property).
110/// - 0.2.0: the head object (`metadata/timelake-head`), from Timelake 0.3 (which wrote it as `2`).
111pub const FORMAT_VERSION: FormatVersion = FormatVersion::new(0, 2, 0);
112
113/// Optional warehouse-level index of table names, for stores that cannot list (plain HTTPS).
114pub const TABLES_INDEX: &str = "timelake-tables.json";
115
116/// An invalid contract property, duration, or partition value.
117#[derive(Debug, Clone, PartialEq, Eq)]
118pub struct FormatError(pub String);
119
120impl FormatError {
121 fn new(msg: impl Into<String>) -> Self {
122 Self(msg.into())
123 }
124}
125
126impl fmt::Display for FormatError {
127 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
128 f.write_str(&self.0)
129 }
130}
131
132impl std::error::Error for FormatError {}
133
134/// `Result` with [`FormatError`].
135pub type Result<T> = std::result::Result<T, FormatError>;