1use std::io::{self, Read, Seek, SeekFrom, Write};
24use std::path::{Path, PathBuf};
25
26use crate::error::ParseError;
27use crate::iterators::ConFrameIterator;
28
29const MAGIC: &[u8; 8] = b"RCIDX\0\0\0";
30const VERSION: u32 = 1;
31const EDGE: u64 = 4096;
32const HEADER_LEN: usize = 8 + 4 + 4 + 8 + 8 + 8 + 8;
33
34#[derive(Debug, Clone, PartialEq, Eq)]
36pub struct FrameIndex {
37 pub offsets: Vec<u64>,
39 pub data_len: u64,
41}
42
43impl FrameIndex {
44 pub fn len(&self) -> usize {
46 self.offsets.len()
47 }
48
49 pub fn is_empty(&self) -> bool {
51 self.offsets.is_empty()
52 }
53
54 pub fn range(&self, i: usize) -> Option<std::ops::Range<usize>> {
56 let start = *self.offsets.get(i)? as usize;
57 let end = self.offsets.get(i + 1).copied().unwrap_or(self.data_len) as usize;
58 Some(start..end)
59 }
60
61 pub fn scan(text: &str) -> Result<Self, ParseError> {
63 Ok(FrameIndex {
64 offsets: frame_start_offsets_checked(text)?
65 .into_iter()
66 .map(|o| o as u64)
67 .collect(),
68 data_len: text.len() as u64,
69 })
70 }
71}
72
73pub fn frame_start_offsets_checked(text: &str) -> Result<Vec<usize>, ParseError> {
78 let mut out = Vec::new();
79 let mut scanner = ConFrameIterator::new(text);
80 loop {
81 let start = scanner.byte_pos();
82 if start >= text.len() || text.as_bytes()[start..].iter().all(u8::is_ascii_whitespace) {
83 break;
84 }
85 out.push(start);
86 match scanner.forward_fast() {
87 Some(Ok(())) => {}
88 Some(Err(e)) => return Err(e),
89 None => return Err(ParseError::IncompleteFrame),
90 }
91 }
92 Ok(out)
93}
94
95const UTF8_SLICE_MIN: usize = 8 << 20;
97
98fn utf8_slice_target(len: usize, index: usize, n_slices: usize) -> usize {
103 let n = n_slices.max(1) as u128;
104 let target = (index as u128) * (len as u128) / n;
105 usize::try_from(target).unwrap_or(len).min(len)
106}
107
108fn reject_with_absolute_utf8(
115 bytes: &[u8],
116 start: usize,
117 err: std::str::Utf8Error,
118) -> Result<(), std::str::Utf8Error> {
119 let end = match err.error_len() {
120 Some(len) => start
121 .saturating_add(err.valid_up_to())
122 .saturating_add(len)
123 .saturating_add(1),
124 None => bytes.len(),
125 };
126 let end = end.min(bytes.len());
127 if let Err(adjusted) = std::str::from_utf8(&bytes[..end]) {
128 return Err(adjusted);
129 }
130 std::str::from_utf8(bytes).map(|_| ())
131}
132
133pub fn validate_utf8_sliced(bytes: &[u8]) -> Result<&str, std::str::Utf8Error> {
138 let n_slices = (bytes.len() / UTF8_SLICE_MIN).max(1);
139 let mut cuts = Vec::with_capacity(n_slices + 1);
140 cuts.push(0usize);
141 for k in 1..n_slices {
142 let target = utf8_slice_target(bytes.len(), k, n_slices);
143 let prev = *cuts.last().unwrap();
144 if target <= prev || target >= bytes.len() {
145 continue;
146 }
147 match memchr::memchr(b'\n', &bytes[target..]) {
148 Some(p) => cuts.push(target + p + 1),
149 None => break,
150 }
151 }
152 cuts.push(bytes.len());
153 cuts.dedup();
154 let check = |w: &[usize]| -> Result<(), std::str::Utf8Error> {
155 match std::str::from_utf8(&bytes[w[0]..w[1]]) {
156 Ok(_) => Ok(()),
157 Err(err) => reject_with_absolute_utf8(bytes, w[0], err),
158 }
159 };
160 #[cfg(feature = "parallel")]
161 {
162 use rayon::prelude::*;
163 let windows: Vec<&[usize]> = cuts.windows(2).collect();
164 windows.par_iter().try_for_each(|w| check(w))?;
165 }
166 #[cfg(not(feature = "parallel"))]
167 {
168 for w in cuts.windows(2) {
169 check(w)?;
170 }
171 }
172 Ok(unsafe { std::str::from_utf8_unchecked(bytes) })
175}
176
177pub fn sidecar_path(path: &Path) -> PathBuf {
179 let mut s = path.as_os_str().to_owned();
180 s.push(".rcidx");
181 PathBuf::from(s)
182}
183
184fn fnv1a(mut h: u64, bytes: &[u8]) -> u64 {
185 for &b in bytes {
186 h ^= b as u64;
187 h = h.wrapping_mul(0x0000_0100_0000_01b3);
188 }
189 h
190}
191const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325;
192
193#[derive(Debug, Clone, Copy, PartialEq, Eq)]
194struct Fingerprint {
195 size: u64,
196 mtime_ns: u64,
197 edges: u64,
198}
199
200fn modified_ns(meta: &std::fs::Metadata) -> u64 {
201 meta.modified()
202 .ok()
203 .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
204 .map(|d| d.as_nanos() as u64)
205 .unwrap_or(0)
206}
207
208fn hash_edges(head: &[u8], tail: &[u8]) -> u64 {
209 fnv1a(fnv1a(FNV_OFFSET, head), tail)
210}
211
212fn edge_hash(bytes: &[u8]) -> u64 {
214 let n = bytes.len();
215 let head_n = (EDGE as usize).min(n);
216 let tail_n = (EDGE as usize).min(n);
217 hash_edges(&bytes[..head_n], &bytes[n - tail_n..])
218}
219
220fn fingerprint(path: &Path) -> io::Result<Fingerprint> {
221 let mut f = std::fs::File::open(path)?;
222 let meta = f.metadata()?;
223 let size = meta.len();
224 let head_n = EDGE.min(size) as usize;
225 let tail_n = EDGE.min(size) as usize;
226 let mut head = vec![0u8; head_n];
227 f.read_exact(&mut head)?;
228 f.seek(SeekFrom::Start(size - tail_n as u64))?;
229 let mut tail = vec![0u8; tail_n];
230 f.read_exact(&mut tail)?;
231 Ok(Fingerprint {
232 size,
233 mtime_ns: modified_ns(&meta),
234 edges: hash_edges(&head, &tail),
235 })
236}
237
238fn fingerprint_if_same(path: &Path, text: &[u8]) -> io::Result<Option<Fingerprint>> {
244 let mut file = std::fs::File::open(path)?;
245 let meta = file.metadata()?;
246 if meta.len() != text.len() as u64 {
247 return Ok(None);
248 }
249 let mut offset = 0usize;
250 let mut buf = [0u8; 64 * 1024];
251 while offset < text.len() {
252 let want = (text.len() - offset).min(buf.len());
253 let n = file.read(&mut buf[..want])?;
254 if n == 0 || text[offset..offset + n] != buf[..n] {
255 return Ok(None);
256 }
257 offset += n;
258 }
259 Ok(Some(Fingerprint {
260 size: meta.len(),
261 mtime_ns: modified_ns(&meta),
262 edges: edge_hash(text),
263 }))
264}
265
266fn encode(fp: Fingerprint, idx: &FrameIndex) -> Vec<u8> {
267 let mut out = Vec::with_capacity(HEADER_LEN + 8 * idx.offsets.len() + 8);
268 out.extend_from_slice(MAGIC);
269 out.extend_from_slice(&VERSION.to_le_bytes());
270 out.extend_from_slice(&0u32.to_le_bytes());
271 out.extend_from_slice(&fp.size.to_le_bytes());
272 out.extend_from_slice(&fp.mtime_ns.to_le_bytes());
273 out.extend_from_slice(&fp.edges.to_le_bytes());
274 out.extend_from_slice(&(idx.offsets.len() as u64).to_le_bytes());
275 for o in &idx.offsets {
276 out.extend_from_slice(&o.to_le_bytes());
277 }
278 let sum = fnv1a(FNV_OFFSET, &out);
279 out.extend_from_slice(&sum.to_le_bytes());
280 out
281}
282
283fn u64_at(b: &[u8], at: usize) -> u64 {
284 u64::from_le_bytes(b[at..at + 8].try_into().unwrap())
285}
286
287fn decode(bytes: &[u8], fp: Fingerprint) -> Option<FrameIndex> {
289 if bytes.len() < HEADER_LEN + 8 || &bytes[..8] != MAGIC {
290 return None;
291 }
292 if u32::from_le_bytes(bytes[8..12].try_into().unwrap()) != VERSION {
293 return None;
294 }
295 let body = &bytes[..bytes.len() - 8];
296 if fnv1a(FNV_OFFSET, body) != u64_at(bytes, bytes.len() - 8) {
297 return None;
298 }
299 let stored = Fingerprint {
300 size: u64_at(bytes, 16),
301 mtime_ns: u64_at(bytes, 24),
302 edges: u64_at(bytes, 32),
303 };
304 if stored != fp {
305 return None;
306 }
307 let n = usize::try_from(u64_at(bytes, 40)).ok()?;
308 let table = n.checked_mul(8)?;
309 let expected = HEADER_LEN.checked_add(table)?;
310 if body.len() != expected {
311 return None;
312 }
313 let offsets: Vec<u64> = (0..n).map(|i| u64_at(bytes, HEADER_LEN + i * 8)).collect();
314 let ordered = offsets.windows(2).all(|w| w[0] < w[1]);
315 if !ordered || offsets.last().is_some_and(|&o| o >= fp.size) {
316 return None;
317 }
318 Some(FrameIndex {
319 offsets,
320 data_len: fp.size,
321 })
322}
323
324pub fn read_sidecar(path: &Path) -> io::Result<Option<FrameIndex>> {
326 let bytes = match std::fs::read(sidecar_path(path)) {
327 Ok(b) => b,
328 Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(None),
329 Err(e) => return Err(e),
330 };
331 Ok(decode(&bytes, fingerprint(path)?))
332}
333
334fn write_encoded(path: &Path, bytes: &[u8]) -> io::Result<()> {
335 let dst = sidecar_path(path);
336 let mut tmp = dst.as_os_str().to_owned();
337 tmp.push(".tmp");
338 let tmp = PathBuf::from(tmp);
339 {
340 let mut f = std::fs::File::create(&tmp)?;
341 f.write_all(bytes)?;
342 f.sync_all()?;
343 }
344 std::fs::rename(&tmp, &dst)
345}
346
347pub fn write_sidecar(path: &Path, idx: &FrameIndex) -> io::Result<()> {
354 let fp = fingerprint(path)?;
355 if fp.size != idx.data_len {
356 return Err(io::Error::new(
357 io::ErrorKind::InvalidInput,
358 "frame index length does not match the data file size",
359 ));
360 }
361 write_encoded(path, &encode(fp, idx))
362}
363
364fn write_sidecar_for_text(path: &Path, text: &str, idx: &FrameIndex) -> io::Result<()> {
366 if idx.data_len != text.len() as u64 {
367 return Err(io::Error::new(
368 io::ErrorKind::InvalidInput,
369 "frame index length does not match the data file size",
370 ));
371 }
372 let Some(fp) = fingerprint_if_same(path, text.as_bytes())? else {
373 return Err(io::Error::new(
374 io::ErrorKind::InvalidInput,
375 "frame index does not match the data file contents",
376 ));
377 };
378 write_encoded(path, &encode(fp, idx))
379}
380
381pub fn load_or_build(path: &Path, text: &str, write_back: bool) -> Result<FrameIndex, ParseError> {
389 if let Ok(Some(idx)) = read_sidecar(path) {
390 if idx.data_len == text.len() as u64 {
391 return Ok(idx);
392 }
393 }
394 let idx = FrameIndex::scan(text)?;
395 if write_back {
396 let _ = write_sidecar_for_text(path, text, &idx);
397 }
398 Ok(idx)
399}
400
401#[cfg(test)]
402mod tests {
403 use super::*;
404
405 fn fixture(n: usize) -> String {
406 let one = std::fs::read_to_string(concat!(
407 env!("CARGO_MANIFEST_DIR"),
408 "/resources/test/tiny_cuh2.con"
409 ))
410 .unwrap();
411 one.repeat(n)
412 }
413
414 #[test]
415 fn checked_scan_matches_legacy_scan() {
416 let text = fixture(7);
417 let legacy = crate::iterators::frame_start_offsets(&text);
418 assert_eq!(frame_start_offsets_checked(&text).unwrap(), legacy);
419 assert_eq!(legacy.len(), 7);
420 }
421
422 #[test]
423 fn checked_scan_accepts_final_line_without_newline() {
424 for n in [1usize, 3] {
425 let text = fixture(n);
426 let bare = text
427 .strip_suffix('\n')
428 .expect("fixture frame ends with a newline");
429 let parsed = crate::iterators::ConFrameIterator::new(bare)
430 .collect::<Result<Vec<_>, _>>()
431 .expect("parser accepts the final line");
432 assert_eq!(parsed.len(), n);
433 let offsets =
434 frame_start_offsets_checked(bare).expect("checked scan accepts the final line");
435 assert_eq!(offsets, crate::iterators::frame_start_offsets(bare));
436 assert_eq!(offsets.len(), n);
437 let mut scanner = crate::iterators::ConFrameIterator::new(bare);
438 for _ in 0..n {
439 scanner
440 .forward_fast()
441 .unwrap()
442 .expect("skip accepts the final line");
443 }
444 assert!(scanner.forward_fast().is_none());
445 }
446 }
447
448 #[test]
449 fn checked_scan_reports_truncation() {
450 let text = fixture(3);
451 let end = text.strip_suffix('\n').unwrap().rfind('\n').unwrap() + 1;
454 let cut = &text[..end];
455 assert!(frame_start_offsets_checked(cut).is_err());
456 assert!(crate::iterators::ConFrameIterator::new(cut).any(|r| r.is_err()));
457 }
458
459 #[test]
460 fn sliced_utf8_matches_std() {
461 let mut s = "é\n".repeat(5 << 20);
462 s.push_str("ü end\n");
463 assert!(validate_utf8_sliced(s.as_bytes()).is_ok());
464 let mut bad = s.into_bytes();
465 let mid = bad.len() / 2;
466 bad[mid] = 0xff;
467 assert!(validate_utf8_sliced(&bad).is_err());
468 }
469
470 #[test]
471 fn sliced_utf8_error_offset_matches_std() {
472 let mut buf = vec![b'\n'; UTF8_SLICE_MIN * 2 + 64];
473 let at = UTF8_SLICE_MIN + UTF8_SLICE_MIN / 2;
474 buf[at] = 0xff;
475 let err = validate_utf8_sliced(&buf).unwrap_err();
476 let std_err = std::str::from_utf8(&buf).unwrap_err();
477 assert_eq!(err.valid_up_to(), std_err.valid_up_to());
478 assert_eq!(err.error_len(), std_err.error_len());
479 assert_eq!(err.valid_up_to(), at);
480 assert_eq!(err.error_len(), Some(1));
481
482 buf[at] = b'\n';
483 buf.push(0xc2);
484 let err = validate_utf8_sliced(&buf).unwrap_err();
485 let std_err = std::str::from_utf8(&buf).unwrap_err();
486 assert_eq!(err.valid_up_to(), std_err.valid_up_to());
487 assert_eq!(err.error_len(), std_err.error_len());
488 assert_eq!(err.valid_up_to(), buf.len() - 1);
489 assert_eq!(err.error_len(), None);
490 }
491
492 #[test]
493 fn sliced_utf8_lead_plus_non_continuation_is_not_truncated() {
494 let mut buf = b"head\n".to_vec();
498 let at = buf.len();
499 buf.push(0xC2);
500 buf.extend_from_slice(b"A tail\n");
501 let err = validate_utf8_sliced(&buf).unwrap_err();
502 let std_err = std::str::from_utf8(&buf).unwrap_err();
503 assert_eq!(err.valid_up_to(), at);
504 assert_eq!(err.valid_up_to(), std_err.valid_up_to());
505 assert_eq!(err.error_len(), Some(1));
506 assert_eq!(err.error_len(), std_err.error_len());
507
508 let mut buf = b"head\n".to_vec();
512 let at = buf.len();
513 buf.extend_from_slice(&[0xE1, 0x80, b'Z']);
514 buf.extend_from_slice(b" tail\n");
515 let err = validate_utf8_sliced(&buf).unwrap_err();
516 let std_err = std::str::from_utf8(&buf).unwrap_err();
517 assert_eq!(err.valid_up_to(), at);
518 assert_eq!(err.valid_up_to(), std_err.valid_up_to());
519 assert_eq!(err.error_len(), Some(2));
520 assert_eq!(err.error_len(), std_err.error_len());
521 }
522
523 #[test]
524 fn utf8_slice_target_stays_inside_huge_buffers() {
525 let len = 20 << 20;
526 let n = (len / UTF8_SLICE_MIN).max(1);
527 for k in 1..n {
528 assert_eq!(utf8_slice_target(len, k, n), k * len / n);
529 assert!(utf8_slice_target(len, k, n) <= len);
530 }
531 let len = usize::MAX - 1024;
532 let n_slices = (len / UTF8_SLICE_MIN).max(2);
533 let k = n_slices - 1;
534 assert!(k.checked_mul(len).is_none());
535 let target = utf8_slice_target(len, k, n_slices);
536 assert!(target <= len);
537 let expected = (k as u128 * len as u128 / n_slices as u128) as usize;
538 assert_eq!(target, expected);
539 }
540
541 #[test]
542 fn sidecar_roundtrip_and_staleness() {
543 let dir = std::env::temp_dir().join(format!("rcidx-test-{}", std::process::id()));
544 std::fs::create_dir_all(&dir).unwrap();
545 let con = dir.join("t.con");
546 let text = fixture(5);
547 std::fs::write(&con, &text).unwrap();
548 let idx = load_or_build(&con, &text, true).unwrap();
549 assert_eq!(read_sidecar(&con).unwrap(), Some(idx.clone()));
550 for i in 0..idx.len() {
551 let r = idx.range(i).unwrap();
552 assert!(
553 crate::iterators::ConFrameIterator::new(&text[r])
554 .next()
555 .unwrap()
556 .is_ok()
557 );
558 }
559 let side = sidecar_path(&con);
561 let mut b = std::fs::read(&side).unwrap();
562 b[HEADER_LEN] ^= 1;
563 std::fs::write(&side, &b).unwrap();
564 assert_eq!(read_sidecar(&con).unwrap(), None);
565 write_sidecar(&con, &idx).unwrap();
567 let text2 = fixture(6);
568 std::fs::write(&con, &text2).unwrap();
569 assert_eq!(read_sidecar(&con).unwrap(), None);
570 assert_eq!(load_or_build(&con, &text2, true).unwrap().len(), 6);
571 let _ = std::fs::remove_dir_all(&dir);
572 }
573
574 #[test]
575 fn edge_hash_matches_file_fingerprint() {
576 let dir = std::env::temp_dir().join(format!("rcidx-edge-{}", std::process::id()));
577 std::fs::create_dir_all(&dir).unwrap();
578 let con = dir.join("t.con");
579 let text = fixture(40);
580 assert!(text.len() > (EDGE as usize) * 2);
581 std::fs::write(&con, &text).unwrap();
582 let fp = fingerprint(&con).unwrap();
583 assert_eq!(fp.size, text.len() as u64);
584 assert_eq!(fp.edges, edge_hash(text.as_bytes()));
585 let _ = std::fs::remove_dir_all(&dir);
586 }
587
588 #[test]
589 fn same_length_middle_change_is_not_published() {
590 let dir = std::env::temp_dir().join(format!("rcidx-stale-{}", std::process::id()));
591 std::fs::create_dir_all(&dir).unwrap();
592 let con = dir.join("t.con");
593 let text = fixture(40);
594 std::fs::write(&con, &text).unwrap();
595 let idx = FrameIndex::scan(&text).unwrap();
596 let mut changed = text.as_bytes().to_vec();
597 let at = (EDGE as usize..changed.len() - EDGE as usize)
598 .find(|&i| changed[i].is_ascii_alphabetic())
599 .expect("middle of the file has a letter");
600 changed[at] = if changed[at] == b'A' { b'B' } else { b'A' };
601 assert_eq!(edge_hash(text.as_bytes()), edge_hash(&changed));
602 assert_ne!(text.as_bytes(), changed.as_slice());
603 std::fs::write(&con, &changed).unwrap();
604 let err = write_sidecar_for_text(&con, &text, &idx).unwrap_err();
605 assert_eq!(err.kind(), io::ErrorKind::InvalidInput);
606 assert_eq!(read_sidecar(&con).unwrap(), None);
607 assert_eq!(
608 load_or_build(&con, &text, true).unwrap().offsets,
609 idx.offsets
610 );
611 assert_eq!(read_sidecar(&con).unwrap(), None);
612 let new_text = std::str::from_utf8(&changed).unwrap();
613 let published = load_or_build(&con, new_text, true).unwrap();
614 assert_eq!(published, FrameIndex::scan(new_text).unwrap());
615 assert_eq!(read_sidecar(&con).unwrap(), Some(published));
616 let _ = std::fs::remove_dir_all(&dir);
617 }
618
619 fn rechecksum(bytes: &mut [u8]) {
620 let body_len = bytes.len() - 8;
621 let sum = fnv1a(FNV_OFFSET, &bytes[..body_len]);
622 bytes[body_len..].copy_from_slice(&sum.to_le_bytes());
623 }
624
625 #[test]
626 fn checksum_valid_short_table_is_rejected() {
627 let text = fixture(2);
628 let idx = FrameIndex::scan(&text).unwrap();
629 let fp = Fingerprint {
630 size: text.len() as u64,
631 mtime_ns: 0,
632 edges: edge_hash(text.as_bytes()),
633 };
634 let mut bytes = encode(fp, &idx);
635 let n = (idx.len() as u64).saturating_add(1);
636 bytes[40..48].copy_from_slice(&n.to_le_bytes());
637 rechecksum(&mut bytes);
638 assert_eq!(decode(&bytes, fp), None);
639 }
640
641 #[test]
642 fn wrapping_frame_count_is_rejected() {
643 let text = fixture(2);
644 let idx = FrameIndex::scan(&text).unwrap();
645 let fp = Fingerprint {
646 size: text.len() as u64,
647 mtime_ns: 0,
648 edges: edge_hash(text.as_bytes()),
649 };
650 let mut bytes = encode(fp, &idx);
651 let payload = bytes.len() - 8 - HEADER_LEN;
652 let n = (1usize << (usize::BITS - 3)).wrapping_add(payload / 8);
653 assert_eq!(8usize.wrapping_mul(n), payload);
654 assert!(n > idx.len());
655 bytes[40..48].copy_from_slice(&(n as u64).to_le_bytes());
656 rechecksum(&mut bytes);
657 assert_eq!(decode(&bytes, fp), None);
658 }
659}