greplm_core/
meta.rs

1//! Index manifest persisted at `.greplm/meta.json`.
2
3use serde::{Deserialize, Serialize};
4use std::path::Path;
5use std::time::{SystemTime, UNIX_EPOCH};
6
7use crate::error::{Error, Result};
8
9/// On-disk format version. Bump when the segment layout changes.
10///
11/// - v2 added the per-segment `seg-N.refs` reference/call-edge table.
12/// - v3 switched the `docs`/`syms`/`refs` side tables from JSON to postcard
13///   (compact binary) to cut on-disk size and cold-start parse time.
14/// - v4 packs each posting list's cardinality into the FST value alongside its
15///   offset, so query planning can intersect rarest-first without touching the
16///   postings blob.
17/// - v5 adds an xxh3 checksum footer to every segment file and the
18///   `pending_tombstones` journal that makes incremental deletes atomic with
19///   the manifest swap.
20/// - v6 switches the `syms`/`refs` side tables to a columnar mmap format
21///   (per-row offsets, doc CSR, packed name columns, persisted name FSTs)
22///   so segment open is O(1) instead of a full decode.
23pub const SCHEMA_VERSION: u32 = 6;
24
25/// Tombstones that are published in the manifest but not yet applied to a
26/// segment's on-disk live bitmap.
27///
28/// An incremental update publishes its new delta segment *and* the doc ids it
29/// supersedes in a single atomic manifest write; the live bitmaps are only
30/// mutated afterwards. Readers subtract any pending tombstones from the live
31/// sets they load, and the next index operation applies and clears the journal
32/// (idempotently), so a crash between the publish and the bitmap writes can
33/// never surface deleted/stale documents nor lose the new ones.
34#[derive(Debug, Clone, Serialize, Deserialize)]
35pub struct PendingTombstones {
36    pub segment_id: u64,
37    pub doc_ids: Vec<u32>,
38}
39
40/// Index-wide manifest describing the set of live segments.
41#[derive(Debug, Clone, Serialize, Deserialize)]
42pub struct Meta {
43    pub schema_version: u32,
44    /// IDs of segments that make up the current index.
45    pub segments: Vec<u64>,
46    /// Monotonic counter for allocating new segment IDs.
47    pub next_segment_id: u64,
48    /// Unix timestamp (seconds) of the last successful index operation.
49    pub last_indexed: u64,
50    /// Total live documents across all segments.
51    pub doc_count: u64,
52    /// Total symbols across all segments.
53    pub symbol_count: u64,
54    /// Git commit sha at the time of indexing (empty if not a repo). Lets
55    /// callers detect that the working tree moved (e.g. a branch switch).
56    #[serde(default)]
57    pub indexed_git_head: String,
58    /// Git branch name at the time of indexing (empty if not a repo).
59    #[serde(default)]
60    pub indexed_branch: String,
61    /// Deletes published with the manifest but not yet applied to live
62    /// bitmaps (see [`PendingTombstones`]). Normally empty; non-empty only in
63    /// the window between an incremental publish and its bitmap writes (or
64    /// after a crash inside that window, until the next index op recovers).
65    #[serde(default, skip_serializing_if = "Vec::is_empty")]
66    pub pending_tombstones: Vec<PendingTombstones>,
67}
68
69impl Default for Meta {
70    fn default() -> Self {
71        Self {
72            schema_version: SCHEMA_VERSION,
73            segments: Vec::new(),
74            next_segment_id: 0,
75            last_indexed: 0,
76            doc_count: 0,
77            symbol_count: 0,
78            indexed_git_head: String::new(),
79            indexed_branch: String::new(),
80            pending_tombstones: Vec::new(),
81        }
82    }
83}
84
85impl Meta {
86    pub fn load(path: &Path) -> Result<Meta> {
87        match std::fs::read(path) {
88            Ok(bytes) => {
89                let meta: Meta = serde_json::from_slice(&bytes)?;
90                if meta.schema_version != SCHEMA_VERSION {
91                    return Err(Error::Corrupt(format!(
92                        "index schema version {} != supported {}; run `greplm index` to rebuild",
93                        meta.schema_version, SCHEMA_VERSION
94                    )));
95                }
96                Ok(meta)
97            }
98            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Meta::default()),
99            Err(e) => Err(Error::io(path, e)),
100        }
101    }
102
103    pub fn save(&self, path: &Path) -> Result<()> {
104        let bytes = serde_json::to_vec_pretty(self)?;
105        crate::fsutil::write_atomic(path, &bytes)
106    }
107
108    /// Record the current git HEAD/branch for the indexed tree (best-effort).
109    pub fn record_git_head(&mut self, root: &Path) {
110        if let Some((sha, branch)) = crate::git::head(root) {
111            self.indexed_git_head = sha;
112            self.indexed_branch = branch;
113        }
114    }
115
116    pub fn touch_now(&mut self) {
117        self.last_indexed = SystemTime::now()
118            .duration_since(UNIX_EPOCH)
119            .map(|d| d.as_secs())
120            .unwrap_or(0);
121    }
122
123    pub fn alloc_segment(&mut self) -> u64 {
124        let id = self.next_segment_id;
125        self.next_segment_id += 1;
126        id
127    }
128}
greplm_core/meta.rs

greplm_core/
meta.rs
