agent_procs/daemon/
log_index.rs

1//! Binary sidecar line index for log files.
2//!
3//! Each log file (e.g. `web.stdout`) has a companion `.idx` file that maps
4//! line numbers to byte offsets, enabling random-access reads without
5//! scanning the entire log.
6//!
7//! ## Format
8//!
9//! ```text
10//! Header (16 bytes):
11//!   magic:    [u8; 4] = b"LIDX"
12//!   version:  u32     = 1       (little-endian)
13//!   seq_base: u64     = first sequence number in this file (little-endian)
14//!
15//! Records (16 bytes each):
16//!   byte_offset: u64  = byte position where this line starts (little-endian)
17//!   seq:         u64  = global sequence number (little-endian)
18//! ```
19
20use std::fs::File;
21use std::io::{self, BufWriter, Read, Seek, SeekFrom, Write};
22use std::path::{Path, PathBuf};
23
24const MAGIC: &[u8; 4] = b"LIDX";
25const VERSION: u32 = 1;
26pub const HEADER_SIZE: u64 = 16;
27pub const RECORD_SIZE: u64 = 16;
28
29/// Returns the index path for a given log path (appends `.idx`).
30pub fn idx_path_for(log_path: &Path) -> PathBuf {
31    let mut s = log_path.as_os_str().to_os_string();
32    s.push(".idx");
33    PathBuf::from(s)
34}
35
36/// A single index record mapping a line to its byte offset and sequence number.
37#[derive(Debug, Clone, Copy, PartialEq)]
38pub struct IndexRecord {
39    pub byte_offset: u64,
40    pub seq: u64,
41}
42
43/// Writes index entries to a `.idx` sidecar file.
44pub struct IndexWriter {
45    writer: BufWriter<File>,
46}
47
48impl IndexWriter {
49    /// Create a new index file, writing the header with the given `seq_base`.
50    pub fn create(path: &Path, seq_base: u64) -> io::Result<Self> {
51        let file = File::create(path)?;
52        let mut writer = BufWriter::new(file);
53        writer.write_all(MAGIC)?;
54        writer.write_all(&VERSION.to_le_bytes())?;
55        writer.write_all(&seq_base.to_le_bytes())?;
56        writer.flush()?;
57        Ok(Self { writer })
58    }
59
60    /// Append a record to the index.
61    pub fn append(&mut self, record: IndexRecord) -> io::Result<()> {
62        self.writer.write_all(&record.byte_offset.to_le_bytes())?;
63        self.writer.write_all(&record.seq.to_le_bytes())?;
64        Ok(())
65    }
66
67    /// Flush buffered writes to disk.
68    pub fn flush(&mut self) -> io::Result<()> {
69        self.writer.flush()
70    }
71}
72
73/// Reads index entries from a `.idx` sidecar file.
74pub struct IndexReader {
75    file: File,
76    line_count: usize,
77    pub seq_base: u64,
78}
79
80impl IndexReader {
81    /// Open an existing index file.  Returns `Ok(None)` if the file is too
82    /// small, has wrong magic, or wrong version.
83    pub fn open(path: &Path) -> io::Result<Option<Self>> {
84        let mut file = File::open(path)?;
85        let file_len = file.metadata()?.len();
86        if file_len < HEADER_SIZE {
87            return Ok(None);
88        }
89
90        let mut header = [0u8; 16];
91        file.read_exact(&mut header)?;
92        if &header[0..4] != MAGIC {
93            return Ok(None);
94        }
95        let version = u32::from_le_bytes(header[4..8].try_into().unwrap());
96        if version != VERSION {
97            return Ok(None);
98        }
99        let seq_base = u64::from_le_bytes(header[8..16].try_into().unwrap());
100
101        let data_bytes = file_len - HEADER_SIZE;
102        let line_count = (data_bytes / RECORD_SIZE) as usize;
103
104        Ok(Some(Self {
105            file,
106            line_count,
107            seq_base,
108        }))
109    }
110
111    /// Number of lines indexed in this file.
112    pub fn line_count(&self) -> usize {
113        self.line_count
114    }
115
116    /// Read a single record by 0-based line number.
117    pub fn read_record(&mut self, line: usize) -> io::Result<IndexRecord> {
118        if line >= self.line_count {
119            return Err(io::Error::new(
120                io::ErrorKind::InvalidInput,
121                format!("line {} out of range (count: {})", line, self.line_count),
122            ));
123        }
124        let offset = HEADER_SIZE + (line as u64) * RECORD_SIZE;
125        self.file.seek(SeekFrom::Start(offset))?;
126        let mut buf = [0u8; 16];
127        self.file.read_exact(&mut buf)?;
128        Ok(IndexRecord {
129            byte_offset: u64::from_le_bytes(buf[0..8].try_into().unwrap()),
130            seq: u64::from_le_bytes(buf[8..16].try_into().unwrap()),
131        })
132    }
133
134    /// Read records `[start..start+count)`, clamped to available lines.
135    pub fn read_range(&mut self, start: usize, count: usize) -> io::Result<Vec<IndexRecord>> {
136        let end = (start + count).min(self.line_count);
137        let actual = end.saturating_sub(start);
138        if actual == 0 {
139            return Ok(Vec::new());
140        }
141        let offset = HEADER_SIZE + (start as u64) * RECORD_SIZE;
142        self.file.seek(SeekFrom::Start(offset))?;
143        let mut buf = vec![0u8; actual * RECORD_SIZE as usize];
144        self.file.read_exact(&mut buf)?;
145        let (chunks, remainder) = buf.as_chunks::<{ RECORD_SIZE as usize }>();
146        debug_assert!(remainder.is_empty());
147        let records = chunks
148            .iter()
149            .map(|chunk| IndexRecord {
150                byte_offset: u64::from_le_bytes(chunk[0..8].try_into().unwrap()),
151                seq: u64::from_le_bytes(chunk[8..16].try_into().unwrap()),
152            })
153            .collect();
154        Ok(records)
155    }
156
157    /// Compute line count from file metadata without reading content.
158    pub fn line_count_from_metadata(path: &Path) -> io::Result<usize> {
159        let meta = std::fs::metadata(path)?;
160        let len = meta.len();
161        if len < HEADER_SIZE {
162            return Ok(0);
163        }
164        Ok(((len - HEADER_SIZE) / RECORD_SIZE) as usize)
165    }
166}
167
168#[cfg(test)]
169mod tests {
170    use super::*;
171
172    #[test]
173    fn test_write_read_roundtrip() {
174        let dir = tempfile::tempdir().unwrap();
175        let path = dir.path().join("test.idx");
176
177        let mut writer = IndexWriter::create(&path, 100).unwrap();
178        writer
179            .append(IndexRecord {
180                byte_offset: 0,
181                seq: 100,
182            })
183            .unwrap();
184        writer
185            .append(IndexRecord {
186                byte_offset: 42,
187                seq: 101,
188            })
189            .unwrap();
190        writer
191            .append(IndexRecord {
192                byte_offset: 100,
193                seq: 102,
194            })
195            .unwrap();
196        writer.flush().unwrap();
197
198        let mut reader = IndexReader::open(&path).unwrap().unwrap();
199        assert_eq!(reader.line_count(), 3);
200        assert_eq!(reader.seq_base, 100);
201
202        assert_eq!(
203            reader.read_record(0).unwrap(),
204            IndexRecord {
205                byte_offset: 0,
206                seq: 100
207            }
208        );
209        assert_eq!(
210            reader.read_record(1).unwrap(),
211            IndexRecord {
212                byte_offset: 42,
213                seq: 101
214            }
215        );
216        assert_eq!(
217            reader.read_record(2).unwrap(),
218            IndexRecord {
219                byte_offset: 100,
220                seq: 102
221            }
222        );
223    }
224
225    #[test]
226    fn test_read_range() {
227        let dir = tempfile::tempdir().unwrap();
228        let path = dir.path().join("test.idx");
229
230        let mut writer = IndexWriter::create(&path, 0).unwrap();
231        for i in 0..10 {
232            writer
233                .append(IndexRecord {
234                    byte_offset: i * 50,
235                    seq: i,
236                })
237                .unwrap();
238        }
239        writer.flush().unwrap();
240
241        let mut reader = IndexReader::open(&path).unwrap().unwrap();
242        assert_eq!(reader.line_count(), 10);
243
244        let range = reader.read_range(3, 4).unwrap();
245        assert_eq!(range.len(), 4);
246        assert_eq!(range[0].seq, 3);
247        assert_eq!(range[3].seq, 6);
248    }
249
250    #[test]
251    fn test_idx_path_for() {
252        assert_eq!(
253            idx_path_for(Path::new("/tmp/logs/web.stdout")),
254            PathBuf::from("/tmp/logs/web.stdout.idx")
255        );
256        assert_eq!(
257            idx_path_for(Path::new("/tmp/logs/web.stdout.1")),
258            PathBuf::from("/tmp/logs/web.stdout.1.idx")
259        );
260    }
261
262    #[test]
263    fn test_empty_file_rejected() {
264        let dir = tempfile::tempdir().unwrap();
265        let path = dir.path().join("tiny.idx");
266        std::fs::write(&path, b"tiny").unwrap();
267        assert!(IndexReader::open(&path).unwrap().is_none());
268    }
269
270    #[test]
271    fn test_wrong_magic_rejected() {
272        let dir = tempfile::tempdir().unwrap();
273        let path = dir.path().join("bad.idx");
274        let mut data = vec![0u8; 16];
275        data[0..4].copy_from_slice(b"XXXX");
276        std::fs::write(&path, &data).unwrap();
277        assert!(IndexReader::open(&path).unwrap().is_none());
278    }
279
280    #[test]
281    fn test_line_count_from_metadata() {
282        let dir = tempfile::tempdir().unwrap();
283        let path = dir.path().join("meta.idx");
284
285        let mut writer = IndexWriter::create(&path, 0).unwrap();
286        writer
287            .append(IndexRecord {
288                byte_offset: 0,
289                seq: 0,
290            })
291            .unwrap();
292        writer
293            .append(IndexRecord {
294                byte_offset: 10,
295                seq: 1,
296            })
297            .unwrap();
298        writer.flush().unwrap();
299
300        assert_eq!(IndexReader::line_count_from_metadata(&path).unwrap(), 2);
301    }
302}
agent_procs/daemon/log_index.rs

agent_procs/daemon/
log_index.rs
