agent_procs/daemon/
log_index.rs

1//! Binary sidecar line index for log files.
2//!
3//! Each log file (e.g. `web.stdout`) has a companion `.idx` file that maps
4//! line numbers to byte offsets, enabling random-access reads without
5//! scanning the entire log.
6//!
7//! ## Format
8//!
9//! ```text
10//! Header (16 bytes):
11//!   magic:    [u8; 4] = b"LIDX"
12//!   version:  u32     = 1       (little-endian)
13//!   seq_base: u64     = first sequence number in this file (little-endian)
14//!
15//! Records (16 bytes each):
16//!   byte_offset: u64  = byte position where this line starts (little-endian)
17//!   seq:         u64  = global sequence number (little-endian)
18//! ```
19
20use std::fs::File;
21use std::io::{self, BufWriter, Read, Seek, SeekFrom, Write};
22use std::path::{Path, PathBuf};
23
24const MAGIC: &[u8; 4] = b"LIDX";
25const VERSION: u32 = 1;
26pub const HEADER_SIZE: u64 = 16;
27pub const RECORD_SIZE: u64 = 16;
28
29/// Returns the index path for a given log path (appends `.idx`).
30pub fn idx_path_for(log_path: &Path) -> PathBuf {
31    let mut s = log_path.as_os_str().to_os_string();
32    s.push(".idx");
33    PathBuf::from(s)
34}
35
36/// A single index record mapping a line to its byte offset and sequence number.
37#[derive(Debug, Clone, Copy, PartialEq)]
38pub struct IndexRecord {
39    pub byte_offset: u64,
40    pub seq: u64,
41}
42
43/// Writes index entries to a `.idx` sidecar file.
44pub struct IndexWriter {
45    writer: BufWriter<File>,
46}
47
48impl IndexWriter {
49    /// Create a new index file, writing the header with the given `seq_base`.
50    pub fn create(path: &Path, seq_base: u64) -> io::Result<Self> {
51        let file = File::create(path)?;
52        let mut writer = BufWriter::new(file);
53        writer.write_all(MAGIC)?;
54        writer.write_all(&VERSION.to_le_bytes())?;
55        writer.write_all(&seq_base.to_le_bytes())?;
56        writer.flush()?;
57        Ok(Self { writer })
58    }
59
60    /// Append a record to the index.
61    pub fn append(&mut self, record: IndexRecord) -> io::Result<()> {
62        self.writer.write_all(&record.byte_offset.to_le_bytes())?;
63        self.writer.write_all(&record.seq.to_le_bytes())?;
64        Ok(())
65    }
66
67    /// Flush buffered writes to disk.
68    pub fn flush(&mut self) -> io::Result<()> {
69        self.writer.flush()
70    }
71}
72
73/// Reads index entries from a `.idx` sidecar file.
74pub struct IndexReader {
75    file: File,
76    line_count: usize,
77    pub seq_base: u64,
78}
79
80impl IndexReader {
81    /// Open an existing index file.  Returns `Ok(None)` if the file is too
82    /// small, has wrong magic, or wrong version.
83    pub fn open(path: &Path) -> io::Result<Option<Self>> {
84        let mut file = File::open(path)?;
85        let file_len = file.metadata()?.len();
86        if file_len < HEADER_SIZE {
87            return Ok(None);
88        }
89
90        let mut header = [0u8; 16];
91        file.read_exact(&mut header)?;
92        if &header[0..4] != MAGIC {
93            return Ok(None);
94        }
95        let version = u32::from_le_bytes(header[4..8].try_into().unwrap());
96        if version != VERSION {
97            return Ok(None);
98        }
99        let seq_base = u64::from_le_bytes(header[8..16].try_into().unwrap());
100
101        let data_bytes = file_len - HEADER_SIZE;
102        let line_count = (data_bytes / RECORD_SIZE) as usize;
103
104        Ok(Some(Self {
105            file,
106            line_count,
107            seq_base,
108        }))
109    }
110
111    /// Number of lines indexed in this file.
112    pub fn line_count(&self) -> usize {
113        self.line_count
114    }
115
116    /// Read a single record by 0-based line number.
117    pub fn read_record(&mut self, line: usize) -> io::Result<IndexRecord> {
118        if line >= self.line_count {
119            return Err(io::Error::new(
120                io::ErrorKind::InvalidInput,
121                format!("line {} out of range (count: {})", line, self.line_count),
122            ));
123        }
124        let offset = HEADER_SIZE + (line as u64) * RECORD_SIZE;
125        self.file.seek(SeekFrom::Start(offset))?;
126        let mut buf = [0u8; 16];
127        self.file.read_exact(&mut buf)?;
128        Ok(IndexRecord {
129            byte_offset: u64::from_le_bytes(buf[0..8].try_into().unwrap()),
130            seq: u64::from_le_bytes(buf[8..16].try_into().unwrap()),
131        })
132    }
133
134    /// Read records `[start..start+count)`, clamped to available lines.
135    pub fn read_range(&mut self, start: usize, count: usize) -> io::Result<Vec<IndexRecord>> {
136        let end = (start + count).min(self.line_count);
137        let actual = end.saturating_sub(start);
138        if actual == 0 {
139            return Ok(Vec::new());
140        }
141        let offset = HEADER_SIZE + (start as u64) * RECORD_SIZE;
142        self.file.seek(SeekFrom::Start(offset))?;
143        let mut buf = vec![0u8; actual * RECORD_SIZE as usize];
144        self.file.read_exact(&mut buf)?;
145        let records = buf
146            .chunks_exact(RECORD_SIZE as usize)
147            .map(|chunk| IndexRecord {
148                byte_offset: u64::from_le_bytes(chunk[0..8].try_into().unwrap()),
149                seq: u64::from_le_bytes(chunk[8..16].try_into().unwrap()),
150            })
151            .collect();
152        Ok(records)
153    }
154
155    /// Compute line count from file metadata without reading content.
156    pub fn line_count_from_metadata(path: &Path) -> io::Result<usize> {
157        let meta = std::fs::metadata(path)?;
158        let len = meta.len();
159        if len < HEADER_SIZE {
160            return Ok(0);
161        }
162        Ok(((len - HEADER_SIZE) / RECORD_SIZE) as usize)
163    }
164}
165
166#[cfg(test)]
167mod tests {
168    use super::*;
169
170    #[test]
171    fn test_write_read_roundtrip() {
172        let dir = tempfile::tempdir().unwrap();
173        let path = dir.path().join("test.idx");
174
175        let mut writer = IndexWriter::create(&path, 100).unwrap();
176        writer
177            .append(IndexRecord {
178                byte_offset: 0,
179                seq: 100,
180            })
181            .unwrap();
182        writer
183            .append(IndexRecord {
184                byte_offset: 42,
185                seq: 101,
186            })
187            .unwrap();
188        writer
189            .append(IndexRecord {
190                byte_offset: 100,
191                seq: 102,
192            })
193            .unwrap();
194        writer.flush().unwrap();
195
196        let mut reader = IndexReader::open(&path).unwrap().unwrap();
197        assert_eq!(reader.line_count(), 3);
198        assert_eq!(reader.seq_base, 100);
199
200        assert_eq!(
201            reader.read_record(0).unwrap(),
202            IndexRecord {
203                byte_offset: 0,
204                seq: 100
205            }
206        );
207        assert_eq!(
208            reader.read_record(1).unwrap(),
209            IndexRecord {
210                byte_offset: 42,
211                seq: 101
212            }
213        );
214        assert_eq!(
215            reader.read_record(2).unwrap(),
216            IndexRecord {
217                byte_offset: 100,
218                seq: 102
219            }
220        );
221    }
222
223    #[test]
224    fn test_read_range() {
225        let dir = tempfile::tempdir().unwrap();
226        let path = dir.path().join("test.idx");
227
228        let mut writer = IndexWriter::create(&path, 0).unwrap();
229        for i in 0..10 {
230            writer
231                .append(IndexRecord {
232                    byte_offset: i * 50,
233                    seq: i,
234                })
235                .unwrap();
236        }
237        writer.flush().unwrap();
238
239        let mut reader = IndexReader::open(&path).unwrap().unwrap();
240        assert_eq!(reader.line_count(), 10);
241
242        let range = reader.read_range(3, 4).unwrap();
243        assert_eq!(range.len(), 4);
244        assert_eq!(range[0].seq, 3);
245        assert_eq!(range[3].seq, 6);
246    }
247
248    #[test]
249    fn test_idx_path_for() {
250        assert_eq!(
251            idx_path_for(Path::new("/tmp/logs/web.stdout")),
252            PathBuf::from("/tmp/logs/web.stdout.idx")
253        );
254        assert_eq!(
255            idx_path_for(Path::new("/tmp/logs/web.stdout.1")),
256            PathBuf::from("/tmp/logs/web.stdout.1.idx")
257        );
258    }
259
260    #[test]
261    fn test_empty_file_rejected() {
262        let dir = tempfile::tempdir().unwrap();
263        let path = dir.path().join("tiny.idx");
264        std::fs::write(&path, b"tiny").unwrap();
265        assert!(IndexReader::open(&path).unwrap().is_none());
266    }
267
268    #[test]
269    fn test_wrong_magic_rejected() {
270        let dir = tempfile::tempdir().unwrap();
271        let path = dir.path().join("bad.idx");
272        let mut data = vec![0u8; 16];
273        data[0..4].copy_from_slice(b"XXXX");
274        std::fs::write(&path, &data).unwrap();
275        assert!(IndexReader::open(&path).unwrap().is_none());
276    }
277
278    #[test]
279    fn test_line_count_from_metadata() {
280        let dir = tempfile::tempdir().unwrap();
281        let path = dir.path().join("meta.idx");
282
283        let mut writer = IndexWriter::create(&path, 0).unwrap();
284        writer
285            .append(IndexRecord {
286                byte_offset: 0,
287                seq: 0,
288            })
289            .unwrap();
290        writer
291            .append(IndexRecord {
292                byte_offset: 10,
293                seq: 1,
294            })
295            .unwrap();
296        writer.flush().unwrap();
297
298        assert_eq!(IndexReader::line_count_from_metadata(&path).unwrap(), 2);
299    }
300}
agent_procs/daemon/log_index.rs

agent_procs/daemon/
log_index.rs
