writerslogic_warm/core/format/
mod.rs

1// Copyright 2024-2026 WritersLogic Contributors
2// SPDX-License-Identifier: Apache-2.0
3//! The persisted format of an WARM store: its version constants, an offline verifier and the
4//! migration framework behind `warm-format`.
5//!
6//! The byte-level specification is `docs/file-format/`; this module is its executable form.
7//! Everything that has a version number is defined here once ([`FormatVersions`]), and
8//! `tests/format_golden.rs` fails when a constant moves without a matching golden store.
9//!
10//! - [`scan`] holds the pure byte-slice readers (frames, manifests, headers). They never
11//!   panic and never allocate from an untrusted length, so the fuzz target can drive them.
12//! - [`verify`] validates a store directory without modifying it.
13//! - [`upgrade`] is the migration registry and the steps that exist today.
14
15use serde::Serialize;
16
17pub mod scan;
18pub mod upgrade;
19pub mod verify;
20
21pub use upgrade::{plan_upgrade, upgrade_store, UpgradePlan, UpgradeReport, UpgradeStep};
22pub use verify::{verify_path, VerifyOptions};
23
24/// Version of the standalone persisted vector update directory.
25pub const VECTOR_UPDATE_FORMAT_VERSION: u32 = 1;
26/// Vector checkpoint manifest magic.
27pub const VECTOR_MANIFEST_MAGIC: &[u8; 8] = b"HMSVUP01";
28/// Full-precision vector row magic.
29pub const VECTOR_ROWS_MAGIC: &[u8; 8] = b"HMSVRW01";
30/// Vector update log magic.
31pub const VECTOR_LOG_MAGIC: &[u8; 8] = b"HMSVLG01";
32
33/// Version of the arena manifest (`vectors_data.bin/format.json`) this build writes.
34pub const STORAGE_FORMAT_CURRENT: u32 = super::storage::STORAGE_FORMAT_VERSION;
35/// Oldest arena manifest version this build opens. Version 1 (no manifest) is read as 2.
36pub const STORAGE_FORMAT_OLDEST: u32 = 2;
37/// `version` field of the `store.json` a new store gets. Schema 2 stores write
38/// [`TRANSACTION_MAGIC`] frames, schema 3 stores [`BINARY_TRANSACTION_MAGIC`] frames; a store
39/// keeps the schema it was created with. Schema 4 stores also write binary frames and may hold
40/// a document checkpoint in place of the frames before it (`docs/file-format/CHECKPOINT.md`).
41/// Schema 5 adds namespace-scoped meaning maintenance, retaining binary frame encoding.
42pub const STORE_SCHEMA_VERSION: u32 = 5;
43/// First schema supporting namespace-scoped meaning maintenance mutations.
44pub const MEANING_MAINTENANCE_SCHEMA_VERSION: u32 = 5;
45/// First `store.json` schema whose journal generations may hold a document checkpoint.
46pub const CHECKPOINT_SCHEMA_VERSION: u32 = 4;
47/// Oldest `store.json` schema this build opens.
48pub const STORE_SCHEMA_OLDEST: u32 = 2;
49/// Prefix of a JSON journal frame payload (schema 2 stores); the digit is the journal format
50/// version.
51pub const TRANSACTION_MAGIC: &[u8] = b"HMS-TXN-2\0";
52/// Prefix of a binary journal frame payload (schema 3 stores).
53pub const BINARY_TRANSACTION_MAGIC: &[u8] = b"HMS-TXN-3\0";
54/// Bytes shared by every journal format version: `HMS-TXN-`, then ASCII digits, then NUL.
55pub const TRANSACTION_MAGIC_PREFIX: &[u8] = b"HMS-TXN-";
56/// Header layout version of `documents.text` this build writes.
57pub const TEXT_SEGMENT_VERSION: u8 = 2;
58/// First four bytes of `documents.text`.
59pub const TEXT_SEGMENT_MAGIC: &[u8; 4] = b"HMST";
60/// Length of the `documents.text` header.
61pub const TEXT_SEGMENT_HEADER_LEN: usize = 8;
62/// Bytes of a journal frame header.
63pub const FRAME_HEADER_LEN: usize = super::storage::HEADER_SIZE;
64/// Version word of the end-of-log marker header.
65pub const END_MARK: u32 = super::storage::END_MARK;
66/// Largest decompressed frame payload.
67pub const MAX_FRAME_PAYLOAD: usize = super::storage::MAX_RAW_FRAME_SIZE;
68/// `magic` of `format.json`.
69pub const MANIFEST_MAGIC: &str = super::storage::FORMAT_MAGIC;
70/// Bytes of one `audit.bin` entry.
71pub const AUDIT_ENTRY_LEN: usize = 105;
72/// Bytes of `encryption.salt`.
73pub const SALT_LEN: usize = 16;
74/// Argon2id memory cost in KiB, iteration count and lanes used to derive the AES-256 key.
75pub const KDF_MEMORY_KIB: u32 = 19 * 1024;
76/// See [`KDF_MEMORY_KIB`].
77pub const KDF_ITERATIONS: u32 = 2;
78/// See [`KDF_MEMORY_KIB`].
79pub const KDF_LANES: u32 = 1;
80
81/// Every version number a store carries, as this build reads and writes them.
82#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
83#[serde(rename_all = "camelCase")]
84pub struct FormatVersions {
85    pub storage_format_current: u32,
86    pub storage_format_oldest: u32,
87    pub store_schema: u32,
88    pub transaction_magic: &'static str,
89    pub text_segment: u8,
90}
91
92/// The versions this build writes.
93pub const CURRENT: FormatVersions = FormatVersions {
94    storage_format_current: STORAGE_FORMAT_CURRENT,
95    storage_format_oldest: STORAGE_FORMAT_OLDEST,
96    store_schema: STORE_SCHEMA_VERSION,
97    transaction_magic: "HMS-TXN-3",
98    text_segment: TEXT_SEGMENT_VERSION,
99};
100
101/// How bad a finding is. The order is the order of precedence for the exit code.
102#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)]
103#[serde(rename_all = "camelCase")]
104pub enum Severity {
105    /// Worth knowing, nothing to do.
106    Info,
107    /// The store opens: the engine repairs it on open (a torn tail) or ignores it (a cache).
108    Warning,
109    /// Data or structure is damaged: the engine refuses to open, or would serve less than
110    /// was committed.
111    Error,
112    /// The store is not damaged but this build cannot read it (a newer format, or one that
113    /// was never openable).
114    Unsupported,
115}
116
117/// One observation about a store.
118#[derive(Debug, Clone, Serialize)]
119#[serde(rename_all = "camelCase")]
120pub struct Finding {
121    pub severity: Severity,
122    /// Stable identifier, listed in `docs/file-format/VERIFY.md`.
123    pub code: &'static str,
124    /// Path relative to the store directory.
125    pub file: String,
126    #[serde(skip_serializing_if = "Option::is_none")]
127    pub offset: Option<u64>,
128    pub message: String,
129}
130
131/// Overall verdict of a verification, and the exit code of `warm-format verify`.
132#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
133#[serde(rename_all = "camelCase")]
134pub enum Verdict {
135    /// 0: every check passed.
136    Ok,
137    /// 10: the store opens; the engine repairs a torn tail on open or discards a bad cache.
138    Recoverable,
139    /// 20: committed data or structure is damaged.
140    Corrupt,
141    /// 30: written by a newer format, or never openable by this build.
142    Unsupported,
143}
144
145impl Verdict {
146    pub fn exit_code(self) -> i32 {
147        match self {
148            Self::Ok => 0,
149            Self::Recoverable => 10,
150            Self::Corrupt => 20,
151            Self::Unsupported => 30,
152        }
153    }
154}
155
156/// Versions read from a store's files; `None` where the file is absent or unreadable.
157#[derive(Debug, Clone, Default, Serialize)]
158#[serde(rename_all = "camelCase")]
159pub struct StoreVersions {
160    #[serde(skip_serializing_if = "Option::is_none")]
161    pub storage_format: Option<u32>,
162    #[serde(skip_serializing_if = "Option::is_none")]
163    pub store_schema: Option<u32>,
164    #[serde(skip_serializing_if = "Option::is_none")]
165    pub text_encoder: Option<String>,
166    #[serde(skip_serializing_if = "Option::is_none")]
167    pub vector_encoder: Option<String>,
168    #[serde(skip_serializing_if = "Option::is_none")]
169    pub encrypted: Option<bool>,
170    #[serde(skip_serializing_if = "Option::is_none")]
171    pub dimensions: Option<u64>,
172    /// Journal format versions seen in frame payloads.
173    #[serde(skip_serializing_if = "Vec::is_empty")]
174    pub transaction_formats: Vec<String>,
175    #[serde(skip_serializing_if = "Option::is_none")]
176    pub text_segment: Option<u8>,
177}
178
179/// Result of verifying one store directory.
180#[derive(Debug, Clone, Serialize)]
181#[serde(rename_all = "camelCase")]
182pub struct Report {
183    pub path: String,
184    pub versions: StoreVersions,
185    pub segments: usize,
186    pub frames: usize,
187    pub mutations: usize,
188    /// Documents live after replaying the journal; absent when payloads were not readable.
189    #[serde(skip_serializing_if = "Option::is_none")]
190    pub live_documents: Option<usize>,
191    #[serde(skip_serializing_if = "Option::is_none")]
192    pub live_vectors: Option<usize>,
193    /// Documents carrying a memory-API stamp (`mem1:` version or `_mem` metadata).
194    #[serde(skip_serializing_if = "Option::is_none")]
195    pub memory_stamped_documents: Option<usize>,
196    /// True when every frame payload was decrypted and authenticated (or the store is not
197    /// encrypted), so the replay figures describe the content.
198    pub payloads_authenticated: bool,
199    /// Checks that ran and passed, by name.
200    pub checks: Vec<&'static str>,
201    pub findings: Vec<Finding>,
202    pub verdict: Verdict,
203    pub exit_code: i32,
204    /// For a server data directory: one report per tenant store, keyed by tenant name.
205    #[serde(skip_serializing_if = "Vec::is_empty")]
206    pub tenants: Vec<TenantReport>,
207}
208
209/// A tenant store inside a server data directory.
210#[derive(Debug, Clone, Serialize)]
211#[serde(rename_all = "camelCase")]
212pub struct TenantReport {
213    pub name: String,
214    pub report: Report,
215}
216
217impl Report {
218    pub(crate) fn new(path: &std::path::Path) -> Self {
219        Self {
220            path: path.display().to_string(),
221            versions: StoreVersions::default(),
222            segments: 0,
223            frames: 0,
224            mutations: 0,
225            live_documents: None,
226            live_vectors: None,
227            memory_stamped_documents: None,
228            payloads_authenticated: false,
229            checks: Vec::new(),
230            findings: Vec::new(),
231            verdict: Verdict::Ok,
232            exit_code: 0,
233            tenants: Vec::new(),
234        }
235    }
236
237    pub(crate) fn add(
238        &mut self,
239        severity: Severity,
240        code: &'static str,
241        file: impl Into<String>,
242        offset: Option<u64>,
243        message: impl Into<String>,
244    ) {
245        self.findings.push(Finding {
246            severity,
247            code,
248            file: file.into(),
249            offset,
250            message: message.into(),
251        });
252    }
253
254    /// Computes the verdict from the findings (and those of every tenant).
255    pub(crate) fn finish(&mut self) {
256        let mut worst = self.findings.iter().map(|f| f.severity).max();
257        for tenant in &self.tenants {
258            worst = worst.max(Some(match tenant.report.verdict {
259                Verdict::Ok => Severity::Info,
260                Verdict::Recoverable => Severity::Warning,
261                Verdict::Corrupt => Severity::Error,
262                Verdict::Unsupported => Severity::Unsupported,
263            }));
264        }
265        self.verdict = match worst {
266            None | Some(Severity::Info) => Verdict::Ok,
267            Some(Severity::Warning) => Verdict::Recoverable,
268            Some(Severity::Error) => Verdict::Corrupt,
269            Some(Severity::Unsupported) => Verdict::Unsupported,
270        };
271        self.exit_code = self.verdict.exit_code();
272    }
273}
274
275#[cfg(test)]
276mod tests {
277    use super::*;
278
279    /// The constants this module republishes are the ones the writers use.
280    #[test]
281    fn constants_match_the_writers() {
282        let dir = tempfile::tempdir().unwrap();
283        let path = dir.path().join("documents.text");
284        crate::core::documents::TextSegment::create(
285            &path,
286            #[cfg(feature = "security")]
287            None,
288        )
289        .unwrap();
290        let bytes = std::fs::read(&path).unwrap();
291        assert_eq!(&bytes[..4], TEXT_SEGMENT_MAGIC);
292        assert_eq!(bytes[4], TEXT_SEGMENT_VERSION);
293        assert_eq!(bytes.len(), TEXT_SEGMENT_HEADER_LEN);
294        assert_eq!(TRANSACTION_MAGIC, b"HMS-TXN-2\0");
295        assert_eq!(BINARY_TRANSACTION_MAGIC, b"HMS-TXN-3\0");
296        assert!(TRANSACTION_MAGIC.starts_with(TRANSACTION_MAGIC_PREFIX));
297        assert!(BINARY_TRANSACTION_MAGIC.starts_with(TRANSACTION_MAGIC_PREFIX));
298        assert_eq!(
299            CURRENT.transaction_magic.as_bytes(),
300            &BINARY_TRANSACTION_MAGIC[..9]
301        );
302        assert_eq!(
303            crate::core::engine::journal::JournalFormat::for_schema(STORE_SCHEMA_VERSION),
304            crate::core::engine::journal::JournalFormat::Binary
305        );
306    }
307}
writerslogic_warm/core/format/mod.rs

writerslogic_warm/core/format/
mod.rs
